Merge pull request #78 from video-db/ankit/videodb-llm-proxy

Ankit/videodb llm proxy

backend/director/agents/composio.py

@@ -12,7 +12,7 @@
 )
 
 from director.tools.composio_tool import composio_tool
-from director.llm.openai import OpenAI
+from director.llm import get_default_llm
 from director.llm.base import LLMResponseStatus
 
 logger = logging.getLogger(__name__)
@@ -34,7 +34,7 @@ def __init__(self, session: Session, **kwargs):
         self.agent_name = "composio"
         self.description = f'The Composio agent is used to run tasks related to apps like {os.getenv("COMPOSIO_APPS")} '
         self.parameters = COMPOSIO_PARAMETERS
-        self.llm = OpenAI()
+        self.llm = get_default_llm()
         super().__init__(session=session, **kwargs)
 
     def run(self, task: str, *args, **kwargs) -> AgentResponse:

backend/director/agents/meme_maker.py

@@ -12,7 +12,7 @@
     VideoData,
 )
 from director.tools.videodb_tool import VideoDBTool
-from director.llm.openai import OpenAI
+from director.llm import get_default_llm
 
 logger = logging.getLogger(__name__)
 
@@ -41,7 +41,7 @@ def __init__(self, session: Session, **kwargs):
         self.agent_name = "meme_maker"
         self.description = "Generates meme clips and images based on user prompts. This agent usages LLM to analyze the transcript and visual content of the video to generate memes."
         self.parameters = MEMEMAKER_PARAMETERS
-        self.llm = OpenAI()
+        self.llm = get_default_llm()
         super().__init__(session=session, **kwargs)
 
     def _chunk_docs(self, docs, chunk_size):

backend/director/agents/pricing.py

@@ -8,7 +8,7 @@
     RoleTypes,
     TextContent,
 )
-from director.llm.openai import OpenAI
+from director.llm import get_default_llm
 
 logger = logging.getLogger(__name__)
 
@@ -94,7 +94,7 @@ def __init__(self, session: Session, **kwargs):
         self.agent_name = "pricing"
         self.description = "Agent to get information about the pricing and usage of VideoDB, it is also helpful for running scenarios to get the estimates."
         self.parameters = self.get_parameters()
-        self.llm = OpenAI()
+        self.llm = get_default_llm()
         super().__init__(session=session, **kwargs)
 
     def run(self, query: str, *args, **kwargs) -> AgentResponse:

backend/director/agents/profanity_remover.py

@@ -13,7 +13,7 @@
     ContextMessage,
     RoleTypes,
 )
-from director.llm.openai import OpenAI
+from director.llm import get_default_llm
 from director.tools.videodb_tool import VideoDBTool
 
 logger = logging.getLogger(__name__)
@@ -33,7 +33,7 @@ def __init__(self, session: Session, **kwargs):
             "if user has not given those optional param of beep_audio_id always try with sending it as None so that defaults are picked from env"
         )
         self.parameters = self.get_parameters()
-        self.llm = OpenAI()
+        self.llm = get_default_llm()
         super().__init__(session=session, **kwargs)
 
     def add_beep(self, videodb_tool, video_id, beep_audio_id, timestamps):

backend/director/agents/prompt_clip.py

@@ -12,7 +12,7 @@
     VideoData,
 )
 from director.tools.videodb_tool import VideoDBTool
-from director.llm.openai import OpenAI
+from director.llm import get_default_llm
 
 logger = logging.getLogger(__name__)
 
@@ -46,7 +46,7 @@ def __init__(self, session: Session, **kwargs):
         self.agent_name = "prompt_clip"
         self.description = "Generates video clips based on user prompts. This agent uses AI to analyze the text of a video transcript and identify sentences relevant to the user prompt for making clips. It then generates video clips based on the identified sentences. Use this tool to create clips based on specific themes or topics from a video."
         self.parameters = PROMPTCLIP_AGENT_PARAMETERS
-        self.llm = OpenAI()
+        self.llm = get_default_llm()
         super().__init__(session=session, **kwargs)
 
     def _chunk_docs(self, docs, chunk_size):

backend/director/agents/search.py

@@ -1,7 +1,7 @@
 import logging
 
 from director.agents.base import BaseAgent, AgentResponse, AgentStatus
-from director.llm.openai import OpenAI
+from director.llm import get_default_llm
 from director.core.session import (
     Session,
     MsgStatus,
@@ -61,7 +61,7 @@ class SearchAgent(BaseAgent):
     def __init__(self, session: Session, **kwargs):
         self.agent_name = "search"
         self.description = "Agent to retreive data from VideoDB collections and videos."
-        self.llm = OpenAI()
+        self.llm = get_default_llm()
         self.parameters = SEARCH_AGENT_PARAMETERS
         super().__init__(session=session, **kwargs)
 

backend/director/agents/slack_agent.py

@@ -10,7 +10,7 @@
     RoleTypes,
 )
 from director.tools.slack import send_message_to_channel
-from director.llm.openai import OpenAI
+from director.llm import get_default_llm
 from director.llm.base import LLMResponseStatus
 
 logger = logging.getLogger(__name__)
@@ -29,7 +29,7 @@ def __init__(self, session: Session, **kwargs):
         self.agent_name = "slack"
         self.description = "Messages to a slack channel"
         self.parameters = self.get_parameters()
-        self.llm = OpenAI()
+        self.llm = get_default_llm()
         super().__init__(session=session, **kwargs)
 
     def run(self, message: str, *args, **kwargs) -> AgentResponse:

backend/director/agents/subtitle.py

@@ -12,7 +12,7 @@
     MsgStatus,
 )
 from director.tools.videodb_tool import VideoDBTool
-from director.llm.openai import OpenAI, OpenaiConfig
+from director.llm import get_default_llm
 
 
 from videodb.asset import VideoAsset, TextAsset, TextStyle
@@ -109,7 +109,7 @@ class SubtitleAgent(BaseAgent):
     def __init__(self, session: Session, **kwargs):
         self.agent_name = "subtitle"
         self.description = "An agent designed to add different languages subtitles to a specified video within VideoDB."
-        self.llm = OpenAI(config=OpenaiConfig(timeout=120))
+        self.llm = get_default_llm()
         self.parameters = SUBTITLE_AGENT_PARAMETERS
         super().__init__(session=session, **kwargs)
 

backend/director/agents/video_summary.py

@@ -2,7 +2,7 @@
 
 from director.agents.base import BaseAgent, AgentResponse, AgentStatus
 from director.core.session import ContextMessage, RoleTypes, TextContent, MsgStatus
-from director.llm.openai import OpenAI
+from director.llm import get_default_llm
 from director.tools.videodb_tool import VideoDBTool
 
 logger = logging.getLogger(__name__)
@@ -12,7 +12,7 @@ class VideoSummaryAgent(BaseAgent):
     def __init__(self, session=None, **kwargs):
         self.agent_name = "video_summary"
         self.description = "This is an agent to summarize the given video of VideoDB, if the user wants a certain kind of summary the prompt is required."
-        self.llm = OpenAI()
+        self.llm = get_default_llm()
         self.parameters = self.get_parameters()
         super().__init__(session=session, **kwargs)
 

backend/director/constants.py

@@ -19,6 +19,7 @@ class LLMType(str, Enum):
 
     OPENAI = "openai"
     ANTHROPIC = "anthropic"
+    VIDEODB_PROXY = "videodb_proxy"
 
 
 class EnvPrefix(str, Enum):

backend/director/core/reasoning.py

@@ -13,7 +13,7 @@
     MsgStatus,
 )
 from director.llm.base import LLMResponse
-from director.llm.openai import OpenAI
+from director.llm import get_default_llm
 
 
 logger = logging.getLogger(__name__)
@@ -64,7 +64,7 @@ def __init__(
         self.session = session
         self.system_prompt = REASONING_SYSTEM_PROMPT
         self.max_iterations = 10
-        self.llm = OpenAI()
+        self.llm = get_default_llm()
         self.agents: List[BaseAgent] = []
         self.stop_flag = False
         self.output_message: OutputMessage = self.session.output_message

backend/director/handler.py

@@ -150,16 +150,11 @@ class ConfigHandler:
     def check(self):
         """Check the configuration of the server."""
         videodb_configured = True if os.getenv("VIDEO_DB_API_KEY") else False
-        openai_key_configured = True if os.getenv("OPENAI_API_KEY") else False
-
-        llm_configured = False
-        if openai_key_configured:
-            llm_configured = True
 
         db = load_db(os.getenv("SERVER_DB_TYPE", "sqlite"))
         db_configured = db.health_check()
         return {
             "videodb_configured": videodb_configured,
-            "llm_configured": llm_configured,
+            "llm_configured": True,
             "db_configured": db_configured,
         }

backend/director/llm/__init__.py

@@ -0,0 +1,23 @@
+import os
+
+from director.constants import LLMType
+
+from director.llm.openai import OpenAI
+from director.llm.anthropic import AnthropicAI
+from director.llm.videodb_proxy import VideoDBProxy
+
+
+def get_default_llm():
+    """Get default LLM"""
+
+    openai = True if os.getenv("OPENAI_API_KEY") else False
+    anthropic = True if os.getenv("ANTHROPIC_API_KEY") else False
+
+    default_llm = os.getenv("DEFAULT_LLM")
+
+    if openai or default_llm == LLMType.OPENAI:
+        return OpenAI()
+    elif anthropic or default_llm == LLMType.ANTHROPIC:
+        return AnthropicAI()
+    else:
+        return VideoDBProxy()