updates to dream_kg distribution (#581)

* fix: component cards and variable renaming * component cards for last_chance_service and dff_user_kg_skill created * changes reflected in pipeline_conf * config variable renamed for user-km * service cards and Dockerfile updated for user-km * chore: out-of-date services deleted from dist * refactor: variable renamed
deeppavlov · Oct 24, 2023 · dd0fdba · dd0fdba
1 parent 6d6d5a7
commit dd0fdba
Show file tree

Hide file tree

Showing 11 changed files with 74 additions and 146 deletions.
diff --git a/annotators/custom_entity_linking/Dockerfile b/annotators/custom_entity_linking/Dockerfile
@@ -21,12 +21,12 @@ ARG LANGUAGE=EN
 ENV LANGUAGE ${LANGUAGE}
 
 ARG CONFIG
-ARG PORT
+ARG SERVICE_PORT
 ARG SRC_DIR
 ARG SED_ARG=" | "
 
 ENV CONFIG=$CONFIG
-ENV PORT=$PORT
+ENV SERVICE_PORT=$SERVICE_PORT
 
 COPY ./annotators/custom_entity_linking/requirements.txt ./requirements.txt
 RUN pip install -r ./requirements.txt

diff --git a/annotators/user_knowledge_memorizer/Dockerfile b/annotators/user_knowledge_memorizer/Dockerfile
@@ -8,14 +8,14 @@ ARG TERMINUSDB_SERVER_PASSWORD
 ARG TERMINUSDB_SERVER_URL
 ARG TERMINUSDB_SERVER_TEAM
 ARG TERMINUSDB_SERVER_DB
-ARG CONFIG
+ARG USER_KM_SERVICE_CONFIG
 
 ENV SERVICE_PORT=$SERVICE_PORT
 ENV TERMINUSDB_SERVER_PASSWORD=$TERMINUSDB_SERVER_PASSWORD
 ENV TERMINUSDB_SERVER_URL=$TERMINUSDB_SERVER_URL
 ENV TERMINUSDB_SERVER_TEAM=$TERMINUSDB_SERVER_TEAM
 ENV TERMINUSDB_SERVER_DB=$TERMINUSDB_SERVER_DB
-ENV CONFIG=$CONFIG
+ENV USER_KM_SERVICE_CONFIG=$USER_KM_SERVICE_CONFIG
 
 RUN pip install -U pip wheel setuptools
 

diff --git a/...tors/user_knowledge_memorizer/config.json → ..._knowledge_memorizer/index_load_path.json b/...tors/user_knowledge_memorizer/config.json → ..._knowledge_memorizer/index_load_path.json
diff --git a/annotators/user_knowledge_memorizer/server.py b/annotators/user_knowledge_memorizer/server.py
@@ -29,7 +29,7 @@
 assert TERMINUSDB_SERVER_PASSWORD, logger.error("TerminusDB server password is not specified")
 TERMINUSDB_SERVER_DB = os.getenv("TERMINUSDB_SERVER_DB")
 TERMINUSDB_SERVER_TEAM = os.getenv("TERMINUSDB_SERVER_TEAM")
-config_path = os.getenv("CONFIG")
+config_path = os.getenv("USER_KM_SERVICE_CONFIG")
 with open(config_path, "r") as config_file:
     config = json.load(config_file)
 index_load_path = Path(os.path.expanduser(config["metadata"]["variables"]["CUSTOM_EL"]))

diff --git a/annotators/user_knowledge_memorizer/service_configs/user-knowledge-memorizer/environment.yml b/annotators/user_knowledge_memorizer/service_configs/user-knowledge-memorizer/environment.yml
@@ -6,4 +6,4 @@ TERMINUSDB_SERVER_URL: http://terminusdb-server:6363
 TERMINUSDB_SERVER_PASSWORD: root
 TERMINUSDB_SERVER_TEAM: admin
 TERMINUSDB_SERVER_DB: user_knowledge_db
-CONFIG: config.json
+USER_KM_SERVICE_CONFIG: index_load_path.json
diff --git a/annotators/user_knowledge_memorizer/service_configs/user-knowledge-memorizer/service.yml b/annotators/user_knowledge_memorizer/service_configs/user-knowledge-memorizer/service.yml
@@ -13,7 +13,7 @@ compose:
       TERMINUSDB_SERVER_PASSWORD: root
       TERMINUSDB_SERVER_TEAM: admin
       TERMINUSDB_SERVER_DB: user_knowledge_db
-      CONFIG: config.json
+      USER_KM_SERVICE_CONFIG: index_load_path.json
     context: ./
     dockerfile: annotators/user_knowledge_memorizer/Dockerfile
   command: flask run -h 0.0.0.0 -p 8027

diff --git a/assistant_dists/dream_kg/dev.yml b/assistant_dists/dream_kg/dev.yml
@@ -11,12 +11,6 @@ services:
       - "~/.deeppavlov/cache:/root/.cache"
     ports:
       - 8128:8128
-  spacy-nounphrases:
-    volumes:
-      - "./annotators/spacy_nounphrases:/src"
-      - "./common:/src/common"
-    ports:
-      - 8006:8006
   sentseg:
     volumes:
       - "./annotators/SentSeg:/src"
@@ -47,11 +41,6 @@ services:
     # # you can use persistent local volume if you need
     # volumes:
     #   - ./venv/data/db_data:/root/data/db
-  spelling-preprocessing:
-    volumes:
-      - "./annotators/spelling_preprocessing:/src"
-    ports:
-      - 8074:8074
   combined-classification:
     volumes:
       - "./common:/src/common"

diff --git a/assistant_dists/dream_kg/docker-compose.override.yml b/assistant_dists/dream_kg/docker-compose.override.yml
@@ -2,12 +2,9 @@ services:
   agent:
     command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/dream_kg/pipeline_conf.json'
     environment:
-      WAIT_HOSTS: "sentence-ranker:8128,
-          spacy-nounphrases:8006, sentseg:8011, ranking-based-response-selector:8002,
-          ner:8021, spelling-preprocessing:8074, entity-linking:8075,
-          combined-classification:8087, entity-detection:8103,
-          property-extraction:8136, custom-entity-linking:8153,
-          terminusdb-server:6363, user-knowledge-memorizer:8027,
+      WAIT_HOSTS: "sentence-ranker:8128, sentseg:8011, ranking-based-response-selector:8002,
+          ner:8021, entity-linking:8075, combined-classification:8087, entity-detection:8103,
+          property-extraction:8136, custom-entity-linking:8153, terminusdb-server:6363, user-knowledge-memorizer:8027,
           dff-user-kg-skill:8028, dff-travel-italy-skill:8025"
       WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-480}
       HIGH_PRIORITY_INTENTS: 1
@@ -35,24 +32,6 @@ services:
         reservations:
           memory: 3G
 
-  spacy-nounphrases:
-    env_file: [ .env ]
-    build:
-      args:
-        SERVICE_PORT: 8006
-        SERVICE_NAME: spacy_nounphrases
-      context: .
-      dockerfile: ./annotators/spacy_nounphrases/Dockerfile
-    command: flask run -h 0.0.0.0 -p 8006
-    environment:
-      - FLASK_APP=server
-    deploy:
-      resources:
-        limits:
-          memory: 256M
-        reservations:
-          memory: 256M
-
   sentseg:
     env_file: [ .env ]
     build:
@@ -132,23 +111,6 @@ services:
         reservations:
           memory: 2.5G
 
-  spelling-preprocessing:
-    env_file: [ .env ]
-    build:
-      args:
-        SERVICE_PORT: 8074
-        SERVICE_NAME: spelling_preprocessing
-      context: ./annotators/spelling_preprocessing/
-    command: flask run -h 0.0.0.0 -p 8074
-    environment:
-      - FLASK_APP=server
-    deploy:
-      resources:
-        limits:
-          memory: 100M
-        reservations:
-          memory: 100M
-
   combined-classification:
     env_file: [ .env ]
     build:
@@ -218,7 +180,7 @@ services:
     build:
       args:
         CONFIG: custom_entity_linking.json
-        PORT: 8153
+        SERVICE_PORT: 8153
         SRC_DIR: annotators/custom_entity_linking
       context: ./
       dockerfile: annotators/custom_entity_linking/Dockerfile
@@ -243,7 +205,7 @@ services:
         TERMINUSDB_SERVER_PASSWORD: root
         TERMINUSDB_SERVER_TEAM: admin
         TERMINUSDB_SERVER_DB: user_knowledge_db
-        CONFIG: config.json
+        USER_KM_SERVICE_CONFIG: index_load_path.json
       context: ./
       dockerfile: annotators/user_knowledge_memorizer/Dockerfile
     command: flask run -h 0.0.0.0 -p 8027

diff --git a/assistant_dists/dream_kg/pipeline_conf.json b/assistant_dists/dream_kg/pipeline_conf.json
@@ -36,7 +36,7 @@
             ],
             "is_enabled": true,
             "source": {
-                "component": "components/sbDcAqiNqxFz.yml",
+                "component": "components/vCZorqdcCVBI.yml",
                 "service": "services/agent_services/service_configs/dream_kg"
             }
         },
@@ -64,27 +64,11 @@
             ],
             "is_enabled": true,
             "source": {
-                "component": "components/rFC0YJOoDFvS.yml",
+                "component": "components/vCZorqdcCVBI.yml",
                 "service": "services/agent_services/service_configs/dream_kg"
             }
         },
         "annotators": {
-            "spelling_preprocessing": {
-                "connector": {
-                    "protocol": "http",
-                    "timeout": 1.0,
-                    "url": "http://spelling-preprocessing:8074/respond"
-                },
-                "dialog_formatter": "state_formatters.dp_formatters:last_utt_dialog",
-                "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
-                "previous_services": [],
-                "state_manager_method": "add_annotation_and_reset_human_attributes_for_first_turn",
-                "is_enabled": true,
-                "source": {
-                    "component": "components/pGxj32ic41pvquRXUdqc7A.yml",
-                    "service": "annotators/spelling_preprocessing/service_configs/spelling-preprocessing"
-                }
-            },
             "sentseg": {
                 "connector": {
                     "protocol": "http",
@@ -93,34 +77,14 @@
                 },
                 "dialog_formatter": "state_formatters.dp_formatters:preproc_last_human_utt_dialog",
                 "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
-                "previous_services": [
-                    "annotators.spelling_preprocessing"
-                ],
+                "previous_services": [],
                 "state_manager_method": "add_annotation",
                 "is_enabled": true,
                 "source": {
                     "component": "components/gM4fEjvVqLlSRRRkQfds2g.yml",
                     "service": "annotators/SentSeg/service_configs/sentseg"
                 }
             },
-            "spacy_nounphrases": {
-                "connector": {
-                    "protocol": "http",
-                    "timeout": 1.0,
-                    "url": "http://spacy-nounphrases:8006/respond"
-                },
-                "dialog_formatter": "state_formatters.dp_formatters:preproc_last_human_utt_dialog",
-                "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
-                "previous_services": [
-                    "annotators.spelling_preprocessing"
-                ],
-                "state_manager_method": "add_annotation",
-                "is_enabled": true,
-                "source": {
-                    "component": "components/dswI5sRZbFPmgcNQKH5iPg.yml",
-                    "service": "annotators/spacy_nounphrases/service_configs/spacy-nounphrases"
-                }
-            },
             "ner": {
                 "connector": {
                     "protocol": "http",
@@ -130,7 +94,6 @@
                 "dialog_formatter": "state_formatters.dp_formatters:ner_formatter_dialog",
                 "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
                 "previous_services": [
-                    "annotators.spelling_preprocessing",
                     "annotators.sentseg"
                 ],
                 "state_manager_method": "add_annotation",
@@ -149,7 +112,6 @@
                 "dialog_formatter": "state_formatters.dp_formatters:entity_detection_formatter_dialog",
                 "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
                 "previous_services": [
-                    "annotators.spelling_preprocessing",
                     "annotators.sentseg"
                 ],
                 "state_manager_method": "add_annotation",
@@ -169,8 +131,7 @@
                 "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
                 "previous_services": [
                     "annotators.ner",
-                    "annotators.entity_detection",
-                    "annotators.spacy_nounphrases"
+                    "annotators.entity_detection"
                 ],
                 "state_manager_method": "add_annotation",
                 "is_enabled": true,
@@ -208,7 +169,6 @@
                 "previous_services": [
                     "annotators.ner",
                     "annotators.entity_detection",
-                    "annotators.spacy_nounphrases",
                     "annotators.property_extraction"
                 ],
                 "state_manager_method": "add_annotation",
@@ -226,9 +186,7 @@
                 },
                 "dialog_formatter": "state_formatters.dp_formatters:preproc_last_human_utt_dialog_w_hist",
                 "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
-                "previous_services": [
-                    "annotators.spelling_preprocessing"
-                ],
+                "previous_services": [],
                 "state_manager_method": "add_annotation",
                 "is_enabled": true,
                 "source": {
@@ -274,24 +232,6 @@
                     "component": "components/3RDNPBdybjBlSQZqcc7nGQ.yml",
                     "service": "annotators/NER_deeppavlov/service_configs/ner"
                 }
-            },
-            "spacy_nounphrases": {
-                "connector": {
-                    "protocol": "http",
-                    "timeout": 1.0,
-                    "url": "http://spacy-nounphrases:8006/respond"
-                },
-                "dialog_formatter": "state_formatters.dp_formatters:last_bot_utt_dialog",
-                "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
-                "previous_services": [
-                    "skills"
-                ],
-                "state_manager_method": "add_annotation_prev_bot_utt",
-                "is_enabled": true,
-                "source": {
-                    "component": "components/dswI5sRZbFPmgcNQKH5iPg.yml",
-                    "service": "annotators/spacy_nounphrases/service_configs/spacy-nounphrases"
-                }
             }
         },
         "response_annotator_selectors": {
@@ -300,8 +240,7 @@
                 "class_name": "skill_selectors.post_annotator_selector.connector:PostAnnotatorSelectorConnector",
                 "annotator_names": [
                     "sentseg",
-                    "ner",
-                    "spacy_nounphrases"
+                    "ner"
                 ]
             },
             "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
@@ -333,24 +272,6 @@
                     "service": "services/sentence_ranker/service_configs/sentence-ranker"
                 }
             },
-            "spacy_nounphrases": {
-                "connector": {
-                    "protocol": "http",
-                    "timeout": 1.0,
-                    "url": "http://spacy-nounphrases:8006/respond_batch"
-                },
-                "dialog_formatter": "state_formatters.dp_formatters:hypotheses_list",
-                "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
-                "previous_services": [
-                    "skills"
-                ],
-                "state_manager_method": "add_hypothesis_annotation_batch",
-                "is_enabled": true,
-                "source": {
-                    "component": "components/dswI5sRZbFPmgcNQKH5iPg.yml",
-                    "service": "annotators/spacy_nounphrases/service_configs/spacy-nounphrases"
-                }
-            },
             "entity_detection": {
                 "connector": {
                     "protocol": "http",

diff --git a/components/pmXHLkbPWpFqq37vtWCuS.yml b/components/pmXHLkbPWpFqq37vtWCuS.yml
@@ -0,0 +1,24 @@
+name: dff_user_kg_skill
+display_name: User KG Skill
+component_type: Script-based w/o NNs
+model_type: Dictionary/Pattern-based
+is_customizable: false
+author: publisher@deeppavlov.ai
+description: Looks up entities detected by Entity Detection in Knowledge Graph and uses these entities in bot's responses
+ram_usage: 128M
+gpu_usage: null
+group: skills
+connector:
+  protocol: http
+  timeout: 2.0
+  url: http://dff-user-kg-skill:8028/respond
+dialog_formatter: state_formatters.dp_formatters:dff_user_kg_skill_formatter
+response_formatter: state_formatters.dp_formatters:skill_with_attributes_formatter_service
+previous_services:
+- skill_selectors
+required_previous_services: null
+state_manager_method: add_hypothesis
+tags: null
+endpoint: respond
+service: skills/dff_user_kg_skill/service_configs/dff-user-kg-skill
+date_created: '2023-06-22T09:45:32'