Final knowledge backend

gui/pages/api/DashboardService.js

@@ -229,7 +229,7 @@ export const getVectorDatabases = () => {
 };
 
 export const getVectorDBDetails = (vectorDBId) => {
-  return api.get(`/vector_dbs/get/db/details/${vectorDBId}`);
+  return api.get(`/vector_dbs/db/details/${vectorDBId}`);
 };
 
 export const deleteVectorDB = (vectorDBId) => {
@@ -269,11 +269,11 @@ export const addUpdateKnowledge = (knowledgeData) => {
 };
 
 export const getValidIndices = () => {
-  return api.get(`/vector_db_indices/get/user/valid_indices`);
+  return api.get(`/vector_db_indices/user/valid_indices`);
 };
 
 export const getValidMarketplaceIndices = (knowledgeName) => {
-  return api.get(`/vector_db_indices/get/marketplace/valid_indices/${knowledgeName}`);
+  return api.get(`/vector_db_indices/marketplace/valid_indices/${knowledgeName}`);
 };
 
 export const fetchKnowledgeTemplateOverview = (knowledgeName) => {

main.py

@@ -42,6 +42,11 @@
 from superagi.controllers.user import router as user_router
 from superagi.controllers.agent_execution_config import router as agent_execution_config
 from superagi.controllers.analytics import router as analytics_router
+from superagi.controllers.knowledges import router as knowledges_router
+from superagi.controllers.knowledge_configs import router as knowledge_configs_router
+from superagi.controllers.vector_dbs import router as vector_dbs_router
+from superagi.controllers.vector_db_indices import router as vector_db_indices_router
+from superagi.controllers.marketplace_stats import router as marketplace_stats_router
 from superagi.helper.tool_helper import register_toolkits, register_marketplace_toolkits
 from superagi.lib.logger import logger
 from superagi.llms.google_palm import GooglePalm
@@ -114,8 +119,12 @@
 app.include_router(twitter_oauth_router, prefix="/twitter")
 app.include_router(agent_execution_config, prefix="/agent_executions_configs")
 app.include_router(analytics_router, prefix="/analytics")
-
 app.include_router(google_oauth_router, prefix="/google")
+app.include_router(knowledges_router, prefix="/knowledges")
+app.include_router(knowledge_configs_router, prefix="/knowledge_configs")
+app.include_router(vector_dbs_router, prefix="/vector_dbs")
+app.include_router(vector_db_indices_router, prefix="/vector_db_indices")
+app.include_router(marketplace_stats_router, prefix="/marketplace")
 
 
 # in production you can use Settings management

migrations/versions/71e3980d55f5_knowledge_and_vector_dbs.py

@@ -0,0 +1,87 @@
+"""Knowledge and Vector dbs
+
+Revision ID: 71e3980d55f5
+Revises: cac478732572
+Create Date: 2023-07-26 07:18:06.492832
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '71e3980d55f5'
+down_revision = 'cac478732572'
+branch_labels = None
+depends_on = None
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('knowledge_configs',
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('knowledge_id', sa.Integer(), nullable=False),
+    sa.Column('key', sa.String(), nullable=True),
+    sa.Column('value', sa.Text(), nullable=True),
+    sa.Column('created_at', sa.DateTime(), nullable=True),
+    sa.Column('updated_at', sa.DateTime(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('knowledges',
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.Column('description', sa.String(), nullable=True),
+    sa.Column('vector_db_index_id', sa.Integer(), nullable=True),
+    sa.Column('organisation_id', sa.Integer(), nullable=True),
+    sa.Column('contributed_by', sa.String(), nullable=True),
+    sa.Column('created_at', sa.DateTime(), nullable=True),
+    sa.Column('updated_at', sa.DateTime(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('marketplace_stats',
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('reference_id', sa.Integer(), nullable=True),
+    sa.Column('reference_name', sa.String(), nullable=True),
+    sa.Column('key', sa.String(), nullable=True),
+    sa.Column('value', sa.Integer(), nullable=True),
+    sa.Column('created_at', sa.DateTime(), nullable=True),
+    sa.Column('updated_at', sa.DateTime(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('vector_db_configs',
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('vector_db_id', sa.Integer(), nullable=False),
+    sa.Column('key', sa.String(), nullable=True),
+    sa.Column('value', sa.Text(), nullable=True),
+    sa.Column('created_at', sa.DateTime(), nullable=True),
+    sa.Column('updated_at', sa.DateTime(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('vector_db_indices',
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.Column('vector_db_id', sa.Integer(), nullable=True),
+    sa.Column('dimensions', sa.Integer(), nullable=True),
+    sa.Column('state', sa.String(), nullable=True),
+    sa.Column('created_at', sa.DateTime(), nullable=True),
+    sa.Column('updated_at', sa.DateTime(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('vector_dbs',
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.Column('db_type', sa.String(), nullable=True),
+    sa.Column('organisation_id', sa.Integer(), nullable=True),
+    sa.Column('created_at', sa.DateTime(), nullable=True),
+    sa.Column('updated_at', sa.DateTime(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('vector_dbs')
+    op.drop_table('vector_db_indices')
+    op.drop_table('vector_db_configs')
+    op.drop_table('knowledges')
+    op.drop_table('knowledge_configs')

superagi/controllers/knowledge_configs.py

@@ -0,0 +1,15 @@
+from fastapi_sqlalchemy import db
+from fastapi import HTTPException, Depends, Query, status
+from fastapi import APIRouter
+from superagi.config.config import get_config
+from superagi.helper.auth import check_auth
+from superagi.models.knowledge_configs import KnowledgeConfigs
+from fastapi_jwt_auth import AuthJWT
+
+router = APIRouter()
+
+@router.get("/marketplace/details/{knowledge_id}")
+def get_marketplace_knowledge_configs(knowledge_id: int, Authorize: AuthJWT = Depends(check_auth)):
+    knowledge_configs = db.session.query(KnowledgeConfigs).filter(KnowledgeConfigs.knowledge_id == knowledge_id).all()
+    return knowledge_configs
+

superagi/controllers/knowledges.py

@@ -0,0 +1,166 @@
+from fastapi_sqlalchemy import db
+from fastapi import HTTPException, Depends, Query, status
+from fastapi import APIRouter
+from datetime import datetime
+from superagi.config.config import get_config
+from superagi.helper.auth import get_user_organisation
+from superagi.models.knowledges import Knowledges
+from superagi.models.marketplace_stats import MarketPlaceStats
+from superagi.models.knowledge_configs import KnowledgeConfigs
+from superagi.models.vector_db_indices import VectordbIndices
+from superagi.models.vector_dbs import Vectordbs
+from superagi.helper.s3_helper import S3Helper
+from superagi.models.vector_db_configs import VectordbConfigs
+from superagi.vector_store.vector_factory import VectorFactory
+from superagi.vector_embeddings.vector_embedding_factory import VectorEmbeddingFactory
+from superagi.helper.time_helper import get_time_difference
+
+router = APIRouter()
+
+@router.get("/get/list")
+def get_knowledge_list(
+    page: int = Query(None, title="Page Number"),
+    organisation = Depends(get_user_organisation)
+):
+    """
+    Get Marketplace Knowledge list.
+
+    Args:
+        page (int, optional): The page number for pagination. Defaults to None.
+
+    Returns:
+        dict: The response containing the marketplace list.
+
+    """
+
+    marketplace_knowledges = Knowledges.fetch_marketplace_list(page)
+    marketplace_knowledges_with_install = Knowledges.get_knowledge_install_details(db.session, marketplace_knowledges, organisation)
+    for knowledge in marketplace_knowledges_with_install:
+        knowledge["install_number"] = MarketPlaceStats.get_knowledge_installation_number(knowledge["id"])
+    return marketplace_knowledges_with_install    
+
+@router.get("/marketplace/list/{page}")
+def get_marketplace_knowledge_list(page: int = 0):
+    organisation_id = int(get_config("MARKETPLACE_ORGANISATION_ID"))
+    page_size = 30
+
+    # Apply search filter if provided
+    query = db.session.query(Knowledges).filter(Knowledges.organisation_id == organisation_id)
+
+    if page < 0:
+        knowledges = query.all()
+    # Paginate the results
+    knowledges = query.offset(page * page_size).limit(page_size).all()
+
+    return knowledges
+
+@router.get("/user/list")
+def get_user_knowledge_list(organisation = Depends(get_user_organisation)):
+    marketplace_knowledges = Knowledges.fetch_marketplace_list(page=-1)
+    user_knowledge_list = Knowledges.get_organisation_knowledges(db.session, organisation)
+    for user_knowledge in user_knowledge_list:
+        if user_knowledge["name"] in [knowledge.name for knowledge in marketplace_knowledges]:
+            user_knowledge["is_marketplace"] = True
+        else:
+            user_knowledge["is_marketplace"] = False
+    return user_knowledge_list
+
+@router.get("/marketplace/get/details/{knowledge_name}")
+def get_knowledge_details(knowledge_name: str):
+    knowledge_data = Knowledges.fetch_knowledge_details_marketplace(knowledge_name)
+    knowledge_config_data = KnowledgeConfigs.fetch_knowledge_config_details_marketplace(knowledge_data["id"])
+    knowledge_data_with_config = knowledge_data | knowledge_config_data
+    knowledge_data_with_config["install_number"] = MarketPlaceStats.get_knowledge_installation_number(knowledge_data_with_config["id"])
+    update_time = str(knowledge_data_with_config["updated_at"])
+    update_time = datetime.strptime(update_time, "%Y-%m-%dT%H:%M:%S.%f")
+    knowledge_data_with_config["updated_at"] = datetime.strftime(update_time, '%d %B %Y')
+    return knowledge_data_with_config
+
+@router.get("/marketplace/details/{knowledge_name}")
+def get_marketplace_knowledge_details(knowledge_name: str):
+    organisation_id = int(get_config("MARKETPLACE_ORGANISATION_ID"))
+    knowledge_details = db.session.query(Knowledges).filter(Knowledges.name == knowledge_name, Knowledges.organisation_id == organisation_id).first()
+    return knowledge_details
+
+@router.get("/user/get/details/{knowledge_id}")
+def get_user_knowledge_details(knowledge_id: int):
+    knowledge_data = Knowledges.get_knowledge_from_id(db.session, knowledge_id)
+    vector_database_index = VectordbIndices.get_vector_index_from_id(db.session, knowledge_data.vector_db_index_id)
+    vector_database = Vectordbs.get_vector_db_from_id(db.session, vector_database_index.vector_db_id)
+    knowledge = {
+        "name": knowledge_data.name,
+        "description": knowledge_data.description,
+        "vector_database_index": {
+            "id": vector_database_index.id,
+            "name": vector_database_index.name
+        },
+        "vector_database": vector_database.name,
+        "installation_type": vector_database_index.state
+    }
+    knowledge_config = KnowledgeConfigs.get_knowledge_config_from_knowledge_id(db.session, knowledge_id)
+    knowledge_data_with_config = knowledge | knowledge_config
+    return knowledge_data_with_config
+
+@router.post("/add_or_update/data")
+def add_update_user_knowledge(knowledge_data: dict, organisation = Depends(get_user_organisation)):
+    knowledge_data["organisation_id"] = organisation.id
+    knowledge_data["contributed_by"] = organisation.name
+    knowledge = Knowledges.add_update_knowledges(db.session, knowledge_data)
+    return {"success": True, "id": knowledge.id}
+
+@router.post("/delete/{knowledge_id}")
+def delete_user_knowledge(knowledge_id: int):
+    try:
+        Knowledges.delete_knowledge(db.session, knowledge_id)
+        return {"success": True}
+    except:
+        return {"success": False}
+
+@router.get("/install/{knowledge_name}/index/{vector_db_index_id}")
+def install_selected_knowledge(knowledge_name: str, vector_db_index_id: int, organisation = Depends(get_user_organisation)):
+    vector_db_index = VectordbIndices.get_vector_index_from_id(db.session, vector_db_index_id)
+    selected_knowledge = Knowledges.fetch_knowledge_details_marketplace(knowledge_name)
+    selected_knowledge_config = KnowledgeConfigs.fetch_knowledge_config_details_marketplace(selected_knowledge['id'])
+    file_chunks = S3Helper().get_json_file(selected_knowledge_config["file_path"])
+    vector = Vectordbs.get_vector_db_from_id(db.session, vector_db_index.vector_db_id)
+    db_creds = VectordbConfigs.get_vector_db_config_from_db_id(db.session, vector.id)
+    upsert_data = VectorEmbeddingFactory.build_vector_storge(vector.db_type).get_vector_embeddings_from_chunks(file_chunks)
+    try:
+        vector_db_storage = VectorFactory.build_vector_storage(vector.db_type, vector_db_index.name, **db_creds)
+        vector_db_storage.add_embeddings_to_vector_db(upsert_data)
+    except:
+        return {"success": False}
+    selected_knowledge_data = {
+        "id": -1,
+        "name": selected_knowledge["name"],
+        "description": selected_knowledge["description"],
+        "index_id": vector_db_index_id,
+        "organisation_id": organisation.id,
+        "contributed_by": selected_knowledge["contributed_by"],
+    }
+    new_knowledge = Knowledges.add_update_knowledges(db.session, selected_knowledge_data)
+    removable_key = 'file_path'
+    selected_knowledge_config.pop(removable_key)
+    configs = selected_knowledge_config
+    KnowledgeConfigs.add_update_knowledge_config(db.session, new_knowledge.id, configs)
+    VectordbIndices.update_vector_index_state(db.session, vector_db_index_id, "Marketplace")
+    install_number = MarketPlaceStats.get_knowledge_installation_number(selected_knowledge["id"])
+    MarketPlaceStats.update_knowledge_install_number(db.session, selected_knowledge["id"], int(install_number) + 1)
+    return {"success": True}
+
+@router.post("/uninstall/{knowledge_name}")
+def uninstall_selected_knowledge(knowledge_name: str, organisation = Depends(get_user_organisation)):
+    knowledge = db.session.query(Knowledges).filter(Knowledges.name == knowledge_name, Knowledges.organisation_id == organisation.id).first()
+    knowledge_config = KnowledgeConfigs.get_knowledge_config_from_knowledge_id(db.session, knowledge.id)
+    vector_ids = eval(knowledge_config["vector_ids"])
+    vector_db_index = VectordbIndices.get_vector_index_from_id(db.session, knowledge.vector_db_index_id)
+    vector = Vectordbs.get_vector_db_from_id(db.session, vector_db_index.vector_db_id)
+    db_creds = VectordbConfigs.get_vector_db_config_from_db_id(db.session, vector.id)
+    try:
+        vector_db_storage = VectorFactory.build_vector_storage(vector.db_type, vector_db_index.name, **db_creds)
+        vector_db_storage.delete_embeddings_from_vector_db(vector_ids)
+    except:
+        return {"success": False}
+    KnowledgeConfigs.delete_knowledge_config(db.session, knowledge.id)
+    Knowledges.delete_knowledge(db.session, knowledge.id)
+    return {"success": True}

superagi/controllers/marketplace_stats.py

@@ -0,0 +1,17 @@
+from fastapi_sqlalchemy import db
+from fastapi import HTTPException, Depends, Query, status
+from fastapi import APIRouter
+from superagi.config.config import get_config
+from superagi.models.marketplace_stats import MarketPlaceStats
+from superagi.models.vector_dbs import Vectordbs
+
+router = APIRouter()
+
+@router.get("/knowledge/downloads/{knowledge_id}")
+def count_knowledge_downloads(knowledge_id: int):
+    download_number = db.session.query(MarketPlaceStats).filter(MarketPlaceStats.reference_id == knowledge_id, MarketPlaceStats.reference_name == "KNOWLEDGE", MarketPlaceStats.key == "download_count").first()
+    if download_number is None:
+        downloads = 0
+    else:
+        downloads = download_number.value
+    return downloads

superagi/controllers/types/agent_with_config.py

@@ -19,3 +19,4 @@ class AgentConfigInput(BaseModel):
     LTM_DB: str
     max_iterations: int
     user_timezone: Optional[str]
+    knowledge: Optional[int]

superagi/controllers/vector_db_indices.py

@@ -0,0 +1,45 @@
+from fastapi_sqlalchemy import db
+from fastapi import HTTPException, Depends, Query
+from fastapi import APIRouter
+from superagi.helper.auth import get_user_organisation
+from superagi.models.vector_dbs import Vectordbs
+from superagi.models.vector_db_indices import VectordbIndices
+from superagi.models.knowledges import Knowledges
+from superagi.models.knowledge_configs import KnowledgeConfigs
+
+router = APIRouter()
+
+@router.get("/marketplace/valid_indices/{knowledge_name}")
+def get_marketplace_valid_indices(knowledge_name: str, organisation = Depends(get_user_organisation)):
+    vector_dbs = Vectordbs.get_vector_db_from_organisation(db.session, organisation)
+    knowledge = Knowledges.fetch_knowledge_details_marketplace(knowledge_name)
+    knowledge_with_config = KnowledgeConfigs.fetch_knowledge_config_details_marketplace(knowledge['id'])
+    pinecone = []
+    qdrant = []
+    for vector_db in vector_dbs:
+        indices =  VectordbIndices.get_vector_indices_from_vectordb(db.session, vector_db.id)
+        for index in indices:
+            data = {"id": index.id, "name": index.name}
+            data["is_valid_dimension"] = True if index.dimensions == int(knowledge_with_config["dimensions"]) else False
+            data["is_valid_state"] = True if index.state != "Custom" else False
+            if vector_db.db_type == "Pinecone":
+                pinecone.append(data)
+            if vector_db.db_type == "Qdrant":
+                qdrant.append(data)
+    return {"pinecone": pinecone, "qdrant": qdrant}
+
+@router.get("/user/valid_indices")
+def get_user_valid_indices(organisation = Depends(get_user_organisation)):
+    vector_dbs = Vectordbs.get_vector_db_from_organisation(db.session, organisation)
+    pinecone = []
+    qdrant = []
+    for vector_db in vector_dbs:
+        indices =  VectordbIndices.get_vector_indices_from_vectordb(db.session, vector_db.id)
+        for index in indices:
+            data = {"id": index.id, "name": index.name}
+            data["is_valid_state"] = True if index.state == "Custom" else False
+            if vector_db.db_type == "Pinecone":
+                pinecone.append(data)
+            if vector_db.db_type == "Qdrant":
+                qdrant.append(data)
+    return {"pinecone": pinecone, "qdrant": qdrant}