diff --git a/.gitignore b/.gitignore index 9397e6cf..e4514c56 100644 --- a/.gitignore +++ b/.gitignore @@ -1,49 +1,43 @@ +# Virtual environments /venv/ -Config/api_keys.ini + +# Configuration files .env +Config/api_keys.ini + +# IDE settings .idea/ + +# Compiled Python files __pycache__/ -Logs/log.txt *.pyc -/hiAGI-Dev.log -AgentForge.log +# Logs +Logs/ +*.log +*Logs/ +*logs/ -src/agentforge.egg-info/ -src/agentforge/persona/chatbot.json -src/agentforge/salience.py +# Database directories +*DB/ +*db/ +# Build and distribution files build/ dist/ +src/agentforge.egg-info/ -Examples/API/Logs/ -Examples/CustomAgents/Logs/ -Examples/DB/ -SalienceBot/Logs/ -SalienceBot/db_path/ -/SalienceBot/DB/ -/SalienceBot/Tests/ -/Examples/KG/DB/ - -/Examples/Dyn/DB/ -/Examples/Dyn/Logs/ -/Examples/Dyn/Tests/ - -Examples/CustomAgents/DB/ -/SalienceBot/path/ - -Examples/Chatbot/DB/ -/SalienceBot/Files/ -/SalienceBot/Workspace/ -/Examples/KG/DB/ -/Sandbox/DB/ - -Sandbox/Logs/ - -docs/.obsidian/ - +# Documentation tools docs/.obsidian/ -Sandbox/Logs/ +# Examples and sandbox +Examples/**/DB/ +Examples/**/Logs/ +Examples/**/Tests/ +sandbox/**/DB/ +sandbox/**/Logs/ +sandbox/**/Tests/ -Sandbox/modules/Logs/ +# Specific files +AgentForge.log +src/agentforge/persona/chatbot.json \ No newline at end of file diff --git a/Contributing.md b/CONTRIBUTING.md similarity index 100% rename from Contributing.md rename to CONTRIBUTING.md diff --git a/Sandbox/.agentforge/prompts/custom/TestoAgent.yaml b/Sandbox/.agentforge/prompts/custom/TestoAgent.yaml deleted file mode 100644 index caa1c085..00000000 --- a/Sandbox/.agentforge/prompts/custom/TestoAgent.yaml +++ /dev/null @@ -1,9 +0,0 @@ -Prompts: - System: - Description: You are a an agent designed to test the functionality of the underlying LLM. Please try to respond to the user to the best of your abilities. - - User: - Request: |+ - {text} - -Persona: dignity \ No newline at end of file diff --git a/Sandbox/.agentforge/settings/models.yaml b/Sandbox/.agentforge/settings/models.yaml deleted file mode 100644 index 8d8b0629..00000000 --- a/Sandbox/.agentforge/settings/models.yaml +++ /dev/null @@ -1,114 +0,0 @@ -# Default settings for all models unless overridden -ModelSettings: -# API: gemini_api -# Model: gemini-flash -# API: openai_api -# Model: omni_model - API: lm_studio_api - Model: LMStudio - Params: # Default parameter values - max_new_tokens: 3000 - temperature: 0.8 - top_p: 0.1 - n: 1 - stop: null - do_sample: true - return_prompt: false - return_metadata: false - typical_p: 0.95 - repetition_penalty: 1.05 - encoder_repetition_penalty: 1.0 - top_k: 40 - min_length: 10 - no_repeat_ngram_size: 0 - num_beams: 1 - penalty_alpha: 0 - length_penalty: 1 - early_stopping: false - pad_token_id: null - eos_token_id: null - use_cache: true - num_return_sequences: 1 - bad_words_ids: null - seed: -1 - -# Library of Models and Parameter Defaults Override -ModelLibrary: - openai_api: - module: "openai" - class: "GPT" - models: - omni_model: - name: gpt-4o - params: # Specific parameters for the model - max_new_tokens: 3500 - smart_model: - name: gpt-4 - smart_fast_model: - name: gpt-4-turbo-2024-04-09 - fast_model: - name: gpt-3.5-turbo - long_fast_model: - name: gpt-3.5-turbo-16k - old_fast_model: - name: gpt-3.5-turbo-0613 - old_long_fast_model: - name: gpt-3.5-turbo-16k-0613 - groq_api: - module: "groq_api" - class: "GroqAPI" - models: - llama31: - name: llama-3.1-70b-versatile - openrouter_api: - module: "openrouter" - class: "OpenRouter" - models: - phi3med: - name: microsoft/phi-3-medium-128k-instruct:free - hermes: - name: nousresearch/hermes-3-llama-3.1-405b - reflection: - name: mattshumer/reflection-70b:free - claude_old: - module: "claude_old" - class: "Claude" - models: - claude: - name: claude-2 - claude3_api: - module: "anthropic" - class: "Claude" - models: - claude-3: - name: claude-3-opus-20240229 - gemini_api: - module: "gemini" - class: "Gemini" - models: - gemini-pro: - name: gemini-1.5-pro - gemini-flash: - name: gemini-1.5-flash - lm_studio_api: - module: "LMStudio" - class: "LMStudio" - models: - LMStudio: - name: lmstudio-community/Meta-Llama-3-8B-Instruct-GGUF - params: - host_url: "http://localhost:1234/v1/chat/completions" - allow_custom_value: True - ollama_api: - module: "ollama" - class: "Ollama" - models: - Llama3.1_70b: - name: "llama3.1:70b" - params: - host_url: "http://localhost:11434/api/generate" - allow_custom_value: True - -# Embedding Library (Not much to see here) -EmbeddingLibrary: - library: sentence_transformers diff --git a/Sandbox/.agentforge/settings/system.yaml b/Sandbox/.agentforge/settings/system.yaml deleted file mode 100644 index 19f6f808..00000000 --- a/Sandbox/.agentforge/settings/system.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Persona Settings -PersonasEnabled: true -Persona: default - -# Storage Settings -StorageEnabled: true -SaveMemory: true # Saving Memory won't work if Storage is disabled -ISOTimeStampMemory: true -UnixTimeStampMemory: true -PersistDirectory: ./DB/ChromaDB # Relative path for persistent storage -DBFreshStart: true # Will wipe storage everytime the system is initialized -Embedding: all-distilroberta-v1 - -# Misc. Settings -OnTheFly: true - -# Logging Settings -Logging: - Enabled: true - Folder: ./Logs - Files: # Log levels: critical, error, warning, info, debug. - AgentForge: debug - ModelIO: debug - Actions: debug - Results: debug - DiscordClient: error - -# Paths the system (agents) have access to read and write -Paths: - Files: ./Files \ No newline at end of file diff --git a/Sandbox/CustomAgents/TestAgent.py b/Sandbox/CustomAgents/TestAgent.py deleted file mode 100644 index 0893aabb..00000000 --- a/Sandbox/CustomAgents/TestAgent.py +++ /dev/null @@ -1,5 +0,0 @@ -from agentforge.agent import Agent - - -class TestAgent(Agent): - pass diff --git a/Sandbox/CustomAgents/TestoAgent.py b/Sandbox/CustomAgents/TestoAgent.py deleted file mode 100644 index 1cf8848b..00000000 --- a/Sandbox/CustomAgents/TestoAgent.py +++ /dev/null @@ -1,5 +0,0 @@ -from agentforge.agent import Agent - - -class TestoAgent(Agent): - pass diff --git a/Sandbox/RunTestAgent.py b/Sandbox/RunTestAgent.py deleted file mode 100644 index b2d3a6a5..00000000 --- a/Sandbox/RunTestAgent.py +++ /dev/null @@ -1,17 +0,0 @@ -from CustomAgents.TestAgent import TestAgent -from CustomAgents.TestoAgent import TestoAgent - -test = TestAgent() - -text = "Hi! I am testing your functionality, is everything nominal and in order from your point of view?" - -result = test.run(text=text) - -print(f"TestAgent Response: {result}") - -# testo = TestoAgent() -# -# result = testo.run(text=text) -# -# print(f"TestoAgent Response: {result}") - diff --git a/Sandbox/modules/TestActions.py b/Sandbox/modules/TestActions.py deleted file mode 100644 index 28a23877..00000000 --- a/Sandbox/modules/TestActions.py +++ /dev/null @@ -1,3 +0,0 @@ -from agentforge.modules.Actions import Action - -test = Action() diff --git a/docs/Agents/AgentClass.md b/docs/Agents/AgentClass.md index 7359b984..8d0e16d2 100644 --- a/docs/Agents/AgentClass.md +++ b/docs/Agents/AgentClass.md @@ -12,6 +12,7 @@ The `Agent` class is designed to: - Provide essential attributes and methods for agent operation. - Facilitate seamless integration with various workflows and data structures. - Simplify the creation of custom agents through method overriding. +- Allow flexible naming of agents by accepting an optional `agent_name` parameter. By subclassing the `Agent` class, developers can create custom agents that inherit default behaviors and override methods to implement specific functionalities. @@ -23,7 +24,7 @@ By subclassing the `Agent` class, developers can create custom agents that inher The `Agent` class utilizes several key attributes: -- **`agent_name`**: The name of the agent, typically set to the class name. +- **`agent_name`**: The name of the agent, set to the provided `agent_name` parameter or defaults to the class name if `agent_name` is not provided. - **`logger`**: A logger instance initialized with the agent’s name for logging messages. - **`config`**: An instance of the `Config` class that handles configuration loading. - **`prompt_handling`**: An instance of the `PromptHandling` class for managing prompt templates. @@ -37,28 +38,38 @@ The `Agent` class utilizes several key attributes: ```python class Agent: - def __init__(self): + def __init__(self, agent_name: Optional[str] = None): """ Initializes an Agent instance, setting up its name, logger, data attributes, and agent-specific configurations. It attempts to load the agent's configuration data and storage settings. + + Args: + name (Optional[str]): The name of the agent. If not provided, the class name is used. """ - self.agent_name: str = self.__class__.__name__ + # Set agent_name to the provided name or default to the class name + self.agent_name: str = agent_name if agent_name is not None else self.__class__.__name__ + + # Initialize logger with the agent's name self.logger: Logger = Logger(name=self.agent_name) + + # Initialize other configurations and handlers self.config = Config() self.prompt_handling = PromptHandling() + # Initialize data attributes self.data: Dict[str, Any] = {} self.prompt: Optional[List[str]] = None self.result: Optional[str] = None self.output: Optional[str] = None + # Initialize agent_data if it hasn't been set already if not hasattr(self, 'agent_data'): # Prevent re-initialization self.agent_data: Optional[Dict[str, Any]] = None ``` **Explanation**: -- **`self.agent_name`**: Automatically set to the class name, ensuring consistency between the agent's name and its class. +- **`self.agent_name`**: Set to the provided `agent_name` parameter if given; otherwise, defaults to the class name. This allows for flexible naming of agents without needing to create separate subclasses. - **`self.logger`**: Initialized with the agent's name for consistent logging. - **`self.config`**: Loads the configuration settings for the agent and the system. - **`self.prompt_handling`**: Manages the rendering and validation of prompt templates. @@ -93,12 +104,12 @@ def run(self, **kwargs: Any) -> Optional[str]: self.logger.log(f"\n{self.agent_name} - Running...", 'info') self.load_data(**kwargs) self.process_data() - self.generate_prompt() + self.render_prompt() self.run_llm() self.parse_result() self.save_to_storage() self.build_output() - self.data = {} + self.template_data = {} self.logger.log(f"\n{self.agent_name} - Done!", 'info') except Exception as e: self.logger.log(f"Agent execution failed: {e}", 'error') @@ -141,7 +152,7 @@ Understanding the following key concepts is essential for effectively utilizing - Includes parameters (`params`) and prompt templates (`prompts`). - Example: ```python - self.data.update({ + self.prompt_data.update({ 'params': self.agent_data.get('params').copy(), 'prompts': self.agent_data['prompts'].copy() }) @@ -155,7 +166,7 @@ Understanding the following key concepts is essential for effectively utilizing if self.agent_data['settings']['system'].get('PersonasEnabled'): persona = self.agent_data.get('persona', {}) for key in persona: - self.data[key.lower()] = persona[key] + self.prompt_data[key.lower()] = persona[key] ``` 3. **Storage Data**: @@ -261,7 +272,16 @@ class CustomAgent(Agent): - **Method**: `self.resolve_storage()` - **Process**: - Checks if storage is enabled in system settings. - - If enabled, initializes the storage instance and stores it in `self.agent_data['storage']`. + - Initializes the storage instance and stores it in `self.agent_data['storage']`, ensuring that `self.agent_data['storage']` exists even if storage is disabled. + - Example: + ```python + def resolve_storage(self): + if not self.agent_data['settings']['system'].get('StorageEnabled'): + self.agent_data['storage'] = None + return + from .utils.ChromaUtils import ChromaUtils + self.agent_data['storage'] = ChromaUtils(self.agent_data['persona']['Name']) + ``` ### Saving to Storage @@ -269,6 +289,7 @@ class CustomAgent(Agent): - **Usage**: - Intended to be overridden to implement specific logic for saving data. - Access the storage instance via `self.agent_data['storage']`. + - Even if storage is disabled, `self.agent_data['storage']` will be `None`, allowing for consistent handling in custom implementations. --- @@ -281,6 +302,7 @@ By understanding the `Agent` class and its core components, you can harness the - **Workflow**: Understand the sequence of methods executed in `run` and how they interact. - **Data Handling**: Utilize `self.data` effectively to manage and manipulate data within your agent. - **Customization**: Override methods as needed to implement custom behaviors. + - **Agent Naming**: Use the `name` parameter to assign custom names to agents without the need for subclassing. --- diff --git a/docs/Agents/AgentMethods.md b/docs/Agents/AgentMethods.md index 188b50f3..fedb5598 100644 --- a/docs/Agents/AgentMethods.md +++ b/docs/Agents/AgentMethods.md @@ -58,14 +58,13 @@ For better understanding, we've grouped the methods into the following categorie ```python from agentforge.agent import Agent -class CustomAgent(Agent): - pass - -agent = CustomAgent() +agent = Agent(agent_name="ExampleAgent") output = agent.run(user_input="Hello, AgentForge!") print(output) ``` +>**Note**: In this case we are assuming we have a `ExampleAgent.yaml` prompt template file in the `.agentforge/prompts/` directory for the agent to use. + --- ## 2. Data Loading Methods @@ -150,7 +149,7 @@ These methods handle loading various types of data into the agent. def load_from_storage(self): collection_name = 'Memories' # Name of the collection in the vector database query = 'User is thinking about planning a trip' # A text query to search the specified collection - self.data['stored_values'] = self.agent_data['storage'].query_memory(collection_name, query) + self.template_data['stored_values'] = self.agent_data['storage'].query_memory(collection_name, query) ``` --- @@ -167,7 +166,7 @@ def load_from_storage(self): ```python def load_additional_data(self): - self.data['timestamp'] = datetime.now().isoformat() + self.template_data['timestamp'] = datetime.now().isoformat() ``` --- @@ -188,7 +187,7 @@ def load_additional_data(self): ```python def load_kwargs(self, **kwargs): - self.data.update(kwargs) + self.template_data.update(kwargs) ``` --- @@ -210,7 +209,7 @@ def load_kwargs(self, **kwargs): ```python def process_data(self): # Convert user input to uppercase - self.data['user_input'] = self.data['user_input'].upper() + self.template_data['user_input'] = self.template_data['user_input'].upper() ``` --- @@ -371,11 +370,12 @@ Let's create a custom agent that performs sentiment analysis on user input using # sentiment_agent.py from agentforge.agent import Agent + class SentimentAgent(Agent): def process_data(self): # Clean the user input by stripping leading/trailing whitespace - self.data['cleaned_input'] = self.data['user_input'].strip() - + self.template_data['cleaned_input'] = self.template_data['user_input'].strip() + def parse_result(self): # Simplify the LLM's response to extract the sentiment response = self.result.lower() @@ -388,7 +388,7 @@ class SentimentAgent(Agent): else: sentiment = 'Undetermined' self.result = sentiment - + def build_output(self): # Build the final output message self.output = f"Sentiment Analysis Result: {self.result}" @@ -414,7 +414,7 @@ Prompts: from sentiment_agent import SentimentAgent # Initialize the agent -agent = SentimentAgent() +agent = SentimentAgent() # The agent name will default to the class name `SentimentAgent` which will load the corresponding prompt template file # User input user_input = " I absolutely love using AgentForge! " diff --git a/docs/Agents/AgentPrompts.md b/docs/Agents/AgentPrompts.md index a597bc55..03188ff5 100644 --- a/docs/Agents/AgentPrompts.md +++ b/docs/Agents/AgentPrompts.md @@ -30,7 +30,7 @@ Each agent requires a corresponding **YAML** prompt file located within the `.ag ### Naming Convention -- **Consistency is Key**: The prompt template **YAML** file **must** have the same name as the agent's class name defined in your code. +- **Match the `agent_name`**: The prompt template **YAML** file **must** have the same name as the agent's `agent_name`. The `agent_name` is determined by the `name` parameter provided during agent initialization or defaults to the agent's class name if no name is given. **Example**: @@ -38,11 +38,16 @@ Each agent requires a corresponding **YAML** prompt file located within the `.ag # echo_agent.py from agentforge.agent import Agent + # Option 1: Using the default class name as agent_name class EchoAgent(Agent): - pass # Agent name is 'EchoAgent' + pass # agent_name defaults to 'EchoAgent' if no name is provided + + # Option 2: Specifying a custom agent_name during initialization + agent = Agent(agent_name="CustomEchoAgent") ``` - - The corresponding prompt file should be named `EchoAgent.yaml` and placed in the `.agentforge/prompts/` directory or any of its subdirectories. + - For `EchoAgent`, the corresponding prompt file should be named `EchoAgent.yaml`. + - For `agent = Agent(agent_name="CustomEchoAgent")`, the prompt file should be named `CustomEchoAgent.yaml`. ### Directory Structure @@ -54,8 +59,9 @@ You can organize your agents and prompt files into subdirectories for better cat .agentforge/ └── prompts/ ├── EchoAgent.yaml + ├── CustomEchoAgent.yaml ├── topic_qanda/ - │ ├── QuestionGeneratorAgent.yaml + │ ├── QuestionGenerator.yaml │ └── AnswerAgent.yaml └── other/ └── HelperAgent.yaml @@ -71,18 +77,17 @@ You can organize your agents and prompt files into subdirectories for better cat Prompt files define the dialogue structures that agents use when interacting with users and LLMs. They are composed of `System` and `User` prompts, each containing one or more **sub-prompts**. ### Basic Structure + The `System` and `User` prompts can be defined in two ways: 1. **As Strings**: Providing the entire prompt template directly as a string. -2. **As a set of Sub-Prompts**: Organizing the prompt into sub-sections for modularity and conditional rendering. - - +2. **As a Set of Sub-Prompts**: Organizing the prompt into sub-sections for modularity and conditional rendering. #### Option 1: Prompts as Strings In the simplest form, you can define the `System` and `User` prompts directly as strings without any sub-prompts. -**Example Prompt File (`SimpleEchoAgent.yaml`):** +**Example Prompt File (`SimpleEcho.yaml`):** ```yaml Prompts: @@ -112,6 +117,7 @@ Prompts: ``` ### Notes: + - **Prompts**: The root key containing the `System` and `User` prompts. - **System Prompt**: Provides the AI assistant with system instructions. - **User Prompt**: Represents the user's input. @@ -127,7 +133,7 @@ Prompts: - **System Prompt**: Contains sub-prompts that define the assistant's behavior, background, or instructions. - **User Prompt**: Contains sub-prompts representing user inputs or specific tasks. -**Example Prompt File (`QuestionGeneratorAgent.yaml`)**: +**Example Prompt File (`QuestionGenerator.yaml`):** ```yaml Prompts: @@ -155,8 +161,8 @@ Prompts: - **User Prompt**: The concatenated and rendered `User` sub-prompts. - **API Handling**: - Depending on the LLM API you are using, the prompts may be sent differently: - - **Separate Prompts**: Some APIs (e.g., OpenAI, LMStudio) accept system and user prompts as separate inputs. - - **Single Prompt**: Other APIs (e.g., Google Gemini) require a single prompt that combines both the system and user messages. + - **Separate Prompts**: Some APIs (e.g., OpenAI) accept system and user prompts as separate inputs. + - **Single Prompt**: Other APIs require a single prompt that combines both the system and user messages. - **Creating Custom APIs**: - The agent provides both the system and user prompts separately in a dictionary variable. - It's up to your API implementation to handle these prompts appropriately, either by sending them as separate messages or concatenating them into a single prompt before sending to the LLM. @@ -205,7 +211,7 @@ Prompts: System: You are a helpful assistant. User: | Please respond in the following format: - + Thoughts: {your thoughts here} Response: {your response here} ``` @@ -249,7 +255,7 @@ Prompts: } ``` -- In this example `/{name/}` and `/{age/}` ensure that `{name}` and `{age}` inside the code snippet is **not** treated as a variable. +- In this example, `/{name/}` and `/{age/}` ensure that `{name}` and `{age}` inside the code snippet are **not** treated as variables. --- @@ -284,9 +290,10 @@ Prompts: **Usage**: ```python -from botty_agent import BottyAgent +from agentforge.agent import Agent -agent = BottyAgent() +# Instantiate the agent with a custom name matching the persona and prompt file +agent = Agent(agent_name="BottyAgent") response = agent.run() print(response) ``` @@ -307,12 +314,25 @@ print(response) Hello! Please introduce yourself. ``` ->Note: This will only work if personas is enabled and BottyAgent is set as the persona either in the system settings or set as an agent override in the corresponding yaml file. - --- ## Important Considerations +### Agent Name and Prompt Matching + +- **`agent_name` Determines Prompt File**: The agent will look for a prompt file matching its `agent_name`. This means that if you provide a custom name during initialization, you must have a corresponding prompt file with the same name. + +**Example**: + +```python +from agentforge.agent import Agent + +# Agent will use 'CustomAgent.yaml' as the prompt file +agent = Agent(agent_name="CustomAgent") +``` + +- Ensure that `CustomAgent.yaml` exists in the `.agentforge/prompts/` directory. + ### Variable Precedence - **Persona vs. Runtime Variables**: Variables provided at runtime override those found in the persona file. @@ -336,6 +356,7 @@ If both the persona file and runtime arguments provide a value for `topic`, the - **Be Mindful of Variable Scope**: Understand where variables come from and how they interact. - **Test Your Prompts**: Regularly test to ensure variables are correctly replaced and prompts render as expected. - **Keep Prompts Clear and Concise**: Write prompts that are easy to read and understand. +- **Match Prompt Files with `agent_name`**: Ensure that the prompt file names match the `agent_name` of your agents. --- @@ -363,9 +384,10 @@ Expertise: Quantum Physics **Usage**: ```python -from knowledge_agent import KnowledgeAgent +from agentforge.agent import Agent -agent = KnowledgeAgent() +# Instantiate the agent with the name matching the prompt and persona files +agent = Agent(agent_name="KnowledgeAgent") response = agent.run(concept="Quantum Entanglement") print(response) ``` @@ -389,14 +411,14 @@ print(response) ## Conclusion -By structuring your prompts using `System` and `User` sections with sub-prompts, you gain precise control over how your agents interact with users and LLMs. Leveraging dynamic variables and persona data allows for rich, context-aware conversations. +By structuring your prompts using `System` and `User` sections with sub-prompts, you gain precise control over how your agents interact with users and LLMs. Leveraging dynamic variables, persona data, and custom `agent_name` allows for rich, context-aware conversations. --- ## Additional Resources - **Prompt Handling Deep Dive**: For a detailed exploration of how prompts are processed, check out the [Prompt Handling Documentation](../Utils/PromptHandling.md). -- **Agents Documentation**: Learn more about creating and customizing agents in the [Agents Guide](Agents.md). +- **Custom Agents Guide**: Learn more about creating and customizing agents, including using custom `agent_name`, in the [Custom Agents Guide](CustomAgents.md). - **Personas**: Understand how to define and use personas in the [Personas Guide](../Personas/Personas.md). --- diff --git a/docs/Agents/CustomAgents.md b/docs/Agents/CustomAgents.md index c79cd107..6a4f94ab 100644 --- a/docs/Agents/CustomAgents.md +++ b/docs/Agents/CustomAgents.md @@ -2,7 +2,7 @@ ## Introduction -Creating custom agents in **AgentForge** allows you to tailor agent behaviors to your specific needs. By subclassing the `Agent` base class, you inherit default functionalities and can override methods to customize behaviors. This guide will walk you through the process of creating and customizing your own agents, as well as how to organize your project for scalability. +Creating custom agents in **AgentForge** allows you to tailor agent behaviors to your specific needs. By subclassing the `Agent` base class, you inherit default functionalities and can override methods to customize behaviors. Additionally, you can assign custom names to agents when instantiating them, enabling the same agent class to use different configurations or prompt templates. This guide will walk you through the process of creating and customizing your own agents, as well as how to organize your project for scalability. --- @@ -10,37 +10,32 @@ Creating custom agents in **AgentForge** allows you to tailor agent behaviors to 1. [Creating a Basic Custom Agent](#1-creating-a-basic-custom-agent) 2. [Creating Agent Prompt Templates](#2-creating-agent-prompt-templates) -3. [Organizing Agents and Project Structure](#3-organizing-agents-and-project-structure) -4. [Using Persona Files](#4-using-persona-files) -5. [Overriding Agent Methods](#5-overriding-agent-methods) -6. [Custom Agent Example](#6-custom-agent-example) -7. [Best Practices](#7-best-practices) -8. [Next Steps](#8-next-steps) +3. [Using Custom Agent Names](#3-using-custom-agent-names) +4. [Organizing Agents and Project Structure](#4-organizing-agents-and-project-structure) +5. [Using Persona Files](#5-using-persona-files) +6. [Overriding Agent Methods](#6-overriding-agent-methods) +7. [Custom Agent Example](#7-custom-agent-example) +8. [Best Practices](#8-best-practices) +9. [Next Steps](#9-next-steps) --- -## 1. Creating a Basic Custom Agent +# 1. Creating Agents in AgentForge -To create a custom agent, you need to define a new Python class that inherits from the `Agent` base class. +In AgentForge, you can create agents in two primary ways: -### Step-by-Step Guide - -**Step 1: Define Your Agent Class** - -Create a Python file for your agent (e.g., `my_custom_agent.py`): +1. **By Instantiating the `Agent` Class (or subclass) with a Custom Name**: This method allows you to create agents by simply specifying a name that matches a prompt template **YAML** file, without writing any additional code. +2. **By Subclassing the `Agent` Class**: Use this method when you need to override or extend the agent's behavior through code by creating a subclass of `Agent`. -```python -from agentforge.agent import Agent +## Method 1: Instantiating the `Agent` Class with a Custom Name -class MyCustomAgent(Agent): - pass # The agent_name is automatically set to 'MyCustomAgent' -``` +This is the simplest and most direct way to create an agent. You provide a custom name when instantiating the `Agent` class, and the agent will use the prompt template **YAML** file that matches this name. No additional code is required. -- By default, `MyCustomAgent` inherits all methods from `Agent`. +### Step-by-Step Guide -**Step 2: Create the Prompt Template** +**Step 1: Create the Prompt Template** -In the `.agentforge/agents/` directory, create a YAML file named `MyCustomAgent.yaml`: +In the `.agentforge/prompts/` directory, create a **YAML** file named `HelpfulAssistant.yaml`: ```yaml Prompts: @@ -49,26 +44,28 @@ Prompts: {user_input} ``` -- The **YAML** file name must exactly match the class name (`MyCustomAgent`) and is case-sensitive. +- The **YAML** file name must match the given `agent_name` exactly and is case-sensitive. -**Step 3: Use Your Custom Agent** +**Step 2: Instantiate the Agent with the Custom Name** -Create a script to run your agent (e.g., `run_my_custom_agent.py`): +Create a script to run your agent (e.g., `run_agent.py`): ```python -from my_custom_agent import MyCustomAgent +from agentforge.agent import Agent -agent = MyCustomAgent() +agent = Agent(agent_name="HelpfulAssistant") response = agent.run(user_input="Hello, AgentForge!") print(response) ``` -**Step 4: Execute the Script** +- By specifying `agent_name="HelpfulAssistant"`, the agent will use the `HelpfulAssistant.yaml` prompt template. + +**Step 3: Execute the Script** Ensure your virtual environment is activated and run the script: ```bash -python run_my_custom_agent.py +python run_agent.py ``` **Example Output:** @@ -77,34 +74,163 @@ python run_my_custom_agent.py Hello! How can I assist you today? ``` -*Note: The actual output depends on the LLM configuration.* +>***Note:** The actual output depends on the LLM configuration.* + +### Benefits + +- **No Code Required**: Create new agents by simply providing a name that matches a prompt template. +- **Flexibility**: Easily switch between different agents by changing the `name` parameter. +- **Efficiency**: Avoids the need to create new subclasses for agents that only differ in their prompt templates. + +### Using Multiple Prompt Templates with the Same Agent Class + +You can create multiple prompt templates and use them with the same `Agent` class by changing the `name` parameter. + +**Example:** + +Create another prompt template named `FriendlyBot.yaml`: + +```yaml +Prompts: + System: You are a friendly chatbot named {bot_name}. + User: | + {user_input} +``` + +Use it in your script: + +```python +from agentforge.agent import Agent + +agent = Agent(name="FriendlyBot") +response = agent.run(user_input="What's the weather today?", bot_name="WeatherBot") +print(response) +``` --- -## 2. Creating Agent Prompt Templates +## Method 2: Subclassing the `Agent` Class to Override Behavior -Prompt templates define how your agent interacts with users and the LLM. They are stored in **YAML** files within the `.agentforge/agents/` directory. +When you need to customize the agent's behavior by overriding methods or adding new functionality, you should create a subclass of the `Agent` class. -### Naming Convention +### Step-by-Step Guide -- The **YAML** file name must match the agent class name exactly (case-sensitive). -- For `MyCustomAgent`, the prompt file is `MyCustomAgent.yaml`. +**Step 1: Define Your Agent Subclass** -### Example Prompt Template +Create a Python file for your agent (e.g., `my_custom_agent.py`): + +```python +from agentforge.agent import Agent + +class MyCustomAgent(Agent): + def process_data(self): + # Custom data processing logic + pass + + def build_output(self): + # Custom output construction + pass +``` + +- The `agent_name` defaults to the class name (`MyCustomAgent`) if no `name` is provided during initialization. +- Override methods to customize the agent's behavior. + +**Step 2: Create the Prompt Template** + +Create a **YAML** file named `MyCustomAgent.yaml` in the `.agentforge/prompts/` directory: ```yaml Prompts: - System: You are a knowledgeable assistant specializing in {specialty}. + System: You are a specialized assistant. User: | {user_input} ``` -- **Variables**: `{specialty}` and `{user_input}` are placeholders that will be replaced at runtime. -- **Sub-Prompts**: `System` and `User` are sub-prompts that structure the conversation. +**Step 3: Use Your Custom Agent** + +Create a script to run your agent (e.g., `run_my_custom_agent.py`): + +```python +from my_custom_agent import MyCustomAgent + +agent = MyCustomAgent() +response = agent.run(user_input="Tell me about AI.") +print(response) +``` + +Since `MyCustomAgent` inherits from the base `Agent` class, you can still provide a custom `name` when initializing it. This allows you to point to a different prompt template, combining the benefits of both methods. By specifying a custom `name`, you can use the overridden methods in your subclass while also utilizing different prompt templates without creating additional subclasses. + +```python +from my_custom_agent import MyCustomAgent + +# Instantiate with a custom name to use a different prompt template +agent = MyCustomAgent(name="SpecializedAssistant") +response = agent.run(user_input="Tell me about AI.") +print(response) +``` + +In this example, `MyCustomAgent` will use the prompt template file `SpecializedAssistant.yaml` instead of `MyCustomAgent.yaml`. This approach allows you to: + +- **Customize Behavior**: Override methods in your subclass to change or extend functionality. +- **Flexible Prompt Templates**: Specify different prompt templates by providing a custom `name` during initialization. +- **Reuse Code**: Avoid duplicating code by not creating additional subclasses for each new prompt template. + +This means you can have a single subclass with custom behavior and easily switch between different prompt templates as needed. + +**Step 4: Execute the Script** + +```bash +python run_my_custom_agent.py +``` + +### When to Use Subclassing + +- **Custom Behavior**: When you need to override or extend the agent's functionality by modifying methods. +- **Complex Logic**: For agents requiring additional processing, data handling, or integration with other systems. +- **Code Reuse**: When building a hierarchy of agents with shared behaviors. + +### Nested Subclassing Example + +You can create nested subclasses to build upon existing behaviors. + +```python +class ParentAgent(Agent): + def process_data(self): + # Parent processing logic + pass + +class ChildAgent(ParentAgent): + def process_data(self): + super().process_data() + # Additional child processing logic + pass + +class GrandchildAgent(ChildAgent): + def process_data(self): + super().process_data() + # Additional grandchild processing logic + pass +``` + +--- + +## Summary of Agent Creation Methods + +### Choosing the Right Method + +- **Instantiating with a Custom Name**: Use this method when you want to create agents that differ only in their prompt templates. It's ideal when no code changes are needed. +- **Subclassing the `Agent` Class**: Use this method when you need to change or extend the agent's behavior through code by overriding methods. + +### Key Points + +- The agent name is determined by the `agent_name` parameter when instantiating the agent. If `agent_name` is not provided, it defaults to the class name. +- The prompt template **YAML** file must match the `agent_name` parameter. +- Avoid creating subclasses solely to use different prompt templates; instead, instantiate the `Agent` class with a custom name. +- Subclassing should be reserved for when you need to override or add methods to change the agent's behavior. --- -## 3. Organizing Agents and Project Structure +## 4. Organizing Agents and Project Structure Proper organization of your agent prompt templates and Python scripts is crucial for **AgentForge** to function correctly. @@ -118,8 +244,9 @@ Proper organization of your agent prompt templates and Python scripts is crucial ### Agent Python Scripts - **Location**: Python scripts (the files containing your agent classes) can be located anywhere within your project's root directory or its subdirectories. -- **Naming Convention**: The **class name** of your agent must **exactly match** the corresponding prompt template **YAML** file name (case-sensitive). - - Example: If your agent class is named `TestAgent`, the prompt template file must be named `TestAgent.yaml`. +- **Naming Convention**: + - The **class name** of your agent does not have to match the prompt template file name if you specify a custom `agent_name` when instantiating the agent. + - If no `agent_name` is provided, it defaults to the class name, and the prompt template file must match the class name. ### Example Structure @@ -128,12 +255,12 @@ your_project/ │ ├── .agentforge/ │ └── prompts/ -│ ├── TestAgent.yaml +│ ├── MyCustomAgent.yaml +│ ├── CustomAgentName.yaml │ └── subdirectory/ │ └── AnotherAgent.yaml -│ ├── custom_agents/ -│ └── test_agent.py +│ └── my_custom_agent.py ├── another_agent.py └── run_agents.py ``` @@ -144,7 +271,7 @@ your_project/ --- -## 4. Using Persona Files +## 5. Using Persona Files Personas provide additional context and information to agents. They are defined in **YAML** files within the `.agentforge/personas/` directory. @@ -177,11 +304,11 @@ Persona: MyCustomAgent - The agent will replace `{Name}`, `{Specialty}`, and `{Background}` with values from the persona file. ->Note: If no persona file is provided to the prompt template the system will default to using the `default.yaml` persona file unless personas is disabled in the system settings. +**Note**: If no persona file is specified, the system will default to using the `default.yaml` persona file unless personas are disabled in the system settings. --- -## 5. Overriding Agent Methods +## 6. Overriding Agent Methods To customize agent behavior, you can override methods inherited from the `Agent` base class. @@ -199,31 +326,32 @@ To customize agent behavior, you can override methods inherited from the `Agent` ```python from agentforge.agent import Agent + class MyCustomAgent(Agent): - def process_data(self): - # Custom data processing - self.data['user_input'] = self.data['user_input'].lower() + def process_data(self): + # Custom data processing + self.template_data['user_input'] = self.template_data['user_input'].lower() - def build_output(self): - # Custom output formatting - self.output = f"Response: {self.result}" + def build_output(self): + # Custom output formatting + self.output = f"Response: {self.result}" ``` - **Calling Base Methods**: Use `super()` to retain base class functionality. ```python - def run(self): + def run(self, **kwargs): # Initial Logic - super().run() + super().run(**kwargs) # Additional processing ``` --- -## 6. Custom Agent Example +## 7. Custom Agent Example -Let's create a custom agent that summarizes a given text and returns the summary. +Let's create a custom agent that summarizes a given text and allows for different summarization styles by using custom agent names. ### Step 1: Define the Custom Agent @@ -231,68 +359,96 @@ Let's create a custom agent that summarizes a given text and returns the summary from agentforge.agent import Agent import json + class SummarizeAgent(Agent): - def parse_result(self): - # Parse the LLM's response as JSON - try: - self.data['parsed_result'] = json.loads(self.result) - except json.JSONDecodeError: - self.data['parsed_result'] = {'summary': self.result} + def parse_result(self): + # Parse the LLM's response as JSON + try: + self.template_data['parsed_result'] = json.loads(self.result) + except json.JSONDecodeError: + self.template_data['parsed_result'] = {'summary': self.result} + + def build_output(self): + summary = self.template_data['parsed_result'].get('summary', 'No summary found.') + self.output = f"Summary:\n{summary}" +``` - def build_output(self): - summary = self.data['parsed_result'].get('summary', 'No summary found.') - self.output = f"Summary:\n{summary}" +### Step 2: Create Different Prompt Templates + +**General Summary (`SummarizeAgent.yaml`):** + +```yaml +Prompts: + System: You are an assistant that summarizes text concisely. + User: | + Please provide a brief summary of the following text in JSON format with the key "summary": + + {text} ``` -### Step 2: Create the Prompt Template (`SummarizeAgent.yaml`) +**Detailed Summary (`DetailedSummarizer.yaml`):** ```yaml Prompts: - System: You are an assistant that summarizes text. + System: You are an assistant that provides detailed summaries. User: | - Please summarize the following text and return the summary in JSON format with the key "summary": + Provide an in-depth summary of the following text, highlighting key points and insights. Return the summary in JSON format with the key "summary": {text} ``` -### Step 3: Run the Agent +### Step 3: Run the Agent with Different Names ```python # run_summarize_agent.py from summarize_agent import SummarizeAgent -agent = SummarizeAgent() text_to_summarize = """ AgentForge is a powerful framework that allows developers to create agents that interact with Large Language Models in a flexible and customizable way. It simplifies the process of building, managing, and deploying AI agents. """ +# Using the general summarizer +agent = SummarizeAgent() response = agent.run(text=text_to_summarize) +print("General Summary:") print(response) + +# Using a detailed summarizer by specifying a custom agent name +detailed_agent = SummarizeAgent(name="DetailedSummarizer") +detailed_response = detailed_agent.run(text=text_to_summarize) +print("\nDetailed Summary:") +print(detailed_response) ``` ### Output -Assuming the LLM returns a response in JSON format: +Assuming the LLM returns appropriate responses: ``` +General Summary: Summary: -AgentForge simplifies building and deploying AI agents by providing a flexible framework for interacting with Large Language Models. +AgentForge simplifies the creation and deployment of AI agents by providing a flexible framework for interacting with Large Language Models. + +Detailed Summary: +Summary: +AgentForge is a comprehensive framework designed to assist developers in building, managing, and deploying AI agents. It offers flexibility and customization, streamlining the interaction with Large Language Models and simplifying complex AI development processes. ``` --- -## 7. Best Practices +## 8. Best Practices -- **Consistent Naming**: Ensure your class name, **YAML** prompt file, and persona file names match exactly. +- **Consistent Naming**: Ensure your `agent_name`, class name, and **YAML** prompt file names match appropriately. + - If you provide a custom `agent_name`, make sure the prompt template file matches this name. - **Use Valid Variable Names**: Variables in prompts and personas should be valid Python identifiers. - **Avoid Name Conflicts**: Be cautious of variable names overlapping between personas and runtime data. - **Test Incrementally**: Test your agent after each change to identify issues early. -- **Leverage Personas**: Use personas to enrich your agent with background information and context. +- **Leverage Custom Names**: Use the `name` parameter to create flexible agents without redundant subclasses. - **Document Your Agent**: Keep notes on custom behaviors and overridden methods for future reference. --- -## 8. Next Steps +## 9. Next Steps - **Explore Agent Methods**: Dive deeper into customizing agents by reading the [Agent Methods Guide](AgentMethods.md). - **Learn About Prompts**: Enhance your prompts by reviewing the [Agent Prompts Guide](AgentPrompts.md). @@ -302,7 +458,7 @@ AgentForge simplifies building and deploying AI agents by providing a flexible f ### **Conclusion** -By following this guide, you can create custom agents tailored to your specific needs. The **AgentForge** framework provides the flexibility to build simple or complex agents, leveraging the power of LLMs with ease. +By following this guide, you can create custom agents tailored to your specific needs. The **AgentForge** framework provides the flexibility to build simple or complex agents, leveraging the power of LLMs with ease. Utilizing the ability to specify custom agent names enhances flexibility and reduces the need for multiple subclasses. --- @@ -313,5 +469,4 @@ If you have questions or need assistance, feel free to reach out: - **Email**: [contact@agentforge.net](mailto:contact@agentforge.net) - **Discord**: Join our [Discord Server](https://discord.gg/ttpXHUtCW6) ---- - +--- \ No newline at end of file diff --git a/docs/LLMs/LLMs.md b/docs/LLMs/LLMs.md index 50ebc02e..b6da5fad 100644 --- a/docs/LLMs/LLMs.md +++ b/docs/LLMs/LLMs.md @@ -41,12 +41,12 @@ def get_llm(self, api: str, model: str): """ try: # Retrieve the model name, module, and class from the 'models.yaml' settings. - model_name = self.data['settings']['models']['ModelLibrary'][api]['models'][model]['name'] - module_name = self.data['settings']['models']['ModelLibrary'][api]['module'] - class_name = self.data['settings']['models']['ModelLibrary'][api]['class'] + model_name = self.template_data['settings']['models']['ModelLibrary'][api]['models'][model]['name'] + module_name = self.template_data['settings']['models']['ModelLibrary'][api]['module'] + class_name = self.template_data['settings']['models']['ModelLibrary'][api]['class'] # Dynamically import the module corresponding to the LLM API. - module = importlib.import_module(f".llm.{module_name}", package=__package__) + module = importlib.import_module(f".apis.{module_name}", package=__package__) # Retrieve the class from the imported module that handles the LLM connection. model_class = getattr(module, class_name) @@ -69,7 +69,7 @@ This approach empowers users to focus on crafting agents and defining their beha ### Directory Structure -In the `llm` [Directory](../../src/agentforge/llm), you will find Python files such as `openai.py`, `anthropic.py`, and `oobabooga.py`. Each file is tailored to interact with its respective LLM API. +In the `llm` [Directory](../../src/agentforge/apis), you will find Python files such as `openai.py`, `anthropic.py`, and `oobabooga.py`. Each file is tailored to interact with its respective LLM API. ### Key Functionality of API Files @@ -86,7 +86,7 @@ class Agent: Executes the language model generation with the generated prompt(s) and any specified parameters. """ try: - model: LLM = self.agent_data['llm'] + model: LLM = self.agent_data['apis'] params = self.agent_data.get("params", {}) params['agent_name'] = self.agent_name self.result = model.generate_text(self.prompt, **params).strip() diff --git a/docs/ToolsAndActions/Actions.md b/docs/ToolsAndActions/Actions.md index 06127368..0947d589 100644 --- a/docs/ToolsAndActions/Actions.md +++ b/docs/ToolsAndActions/Actions.md @@ -172,7 +172,7 @@ Initializes the **Actions** class, setting up logging, storage utilities, and lo **Example:** ```python -from agentforge.modules.Actions import Actions +from agentforge.modules.actions import Actions actions = Actions() ``` @@ -190,7 +190,7 @@ Initializes a specified collection in the vector database with preloaded data. **Example:** ```python -from agentforge.modules.Actions import Actions +from agentforge.modules.actions import Actions actions = Actions() actions.initialize_collection('Actions') @@ -212,7 +212,7 @@ Automatically executes actions for the given objective and context. **Example:** ```python -from agentforge.modules.Actions import Actions +from agentforge.modules.actions import Actions objective = 'Stay up to date with current world events' @@ -242,7 +242,7 @@ Loads actions based on the current objective and specified criteria. **Example:** ```python -from agentforge.modules.Actions import Actions +from agentforge.modules.actions import Actions objective = 'Stay up to date with current world events' @@ -288,26 +288,26 @@ Selects an action for the given objective from the provided action list. **Example:** ```python -from agentforge.modules.Actions import Actions +from agentforge.modules.actions import Actions objective = 'Stay up to date with current world events' context = 'Focus on technology' action_list = { - "Web Search": { - "Description": "The 'Web Search' action combines a Google search, web scraping, and text chunking operations...", - "Example": "# Example usage of the Web Search action:\nquery = \"OpenAI GPT-4\"\n...", - "Instruction": "To perform the 'Web Search' action, follow these steps:\n1. Use the 'Google Search' tool...", - "Name": "Web Search", - "Tools": "Google Search, Web Scrape, Intelligent Chunk", - }, - "Write File": { - "Description": "The 'Write File' action combines directory examination and file writing operations...", - "Example": "# Example usage of the Write File action:\ndirectory_structure = read_directory('path/to/folder'...", - "Instruction": "To perform the 'Write File' action, follow these steps:\n1. Use the 'Read Directory' tool...", - "Name": "Write File", - "Tools": "Read Directory, File Writer", - }, + "Web Search": { + "Description": "The 'Web Search' action combines a Google search, web scraping, and text chunking operations...", + "Example": "# Example usage of the Web Search action:\nquery = \"OpenAI GPT-4\"\n...", + "Instruction": "To perform the 'Web Search' action, follow these steps:\n1. Use the 'Google Search' tool...", + "Name": "Web Search", + "Tools": "Google Search, Web Scrape, Intelligent Chunk", + }, + "Write File": { + "Description": "The 'Write File' action combines directory examination and file writing operations...", + "Example": "# Example usage of the Write File action:\ndirectory_structure = read_directory('path/to/folder'...", + "Instruction": "To perform the 'Write File' action, follow these steps:\n1. Use the 'Read Directory' tool...", + "Name": "Write File", + "Tools": "Read Directory, File Writer", + }, } actions = Actions() @@ -341,19 +341,19 @@ Crafts a new action for the given objective. **Example:** ```python -from agentforge.modules.Actions import Actions +from agentforge.modules.actions import Actions objective = 'Automate file backup' tool_list = { - "File Writer": { - "Args": "folder (str), file (str), text (str), mode (str='a')", - "Command": "write_file", - "Description": "The 'File Writer' tool writes the provided text to a specified file within ...", - "Example": "write_file(folder='/path/to/folder', file='example.txt', text='Hello, World!', mode='a') ...", - "Instruction": "To use the 'File Writer' tool, follow these steps:\n1. Call the `write_file` function ...", - "Name": "File Writer", - }, - # ... more tools ... + "File Writer": { + "Args": "folder (str), file (str), text (str), mode (str='a')", + "Command": "write_file", + "Description": "The 'File Writer' tool writes the provided text to a specified file within ...", + "Example": "write_file(folder='/path/to/folder', file='example.txt', text='Hello, World!', mode='a') ...", + "Instruction": "To use the 'File Writer' tool, follow these steps:\n1. Call the `write_file` function ...", + "Name": "File Writer", + }, + # ... more tools ... } actions = Actions() @@ -392,7 +392,7 @@ Prepares the tool for execution by running the ToolPrimingAgent. **Example:** ```python -from agentforge.modules.Actions import Actions +from agentforge.modules.actions import Actions objective = 'Automate file backup' action = { @@ -446,7 +446,7 @@ Runs the specified tools in sequence for the given objective and action by runni **Example:** ```python -from agentforge.modules.Actions import Actions +from agentforge.modules.actions import Actions objective = 'Automate file backup' action = { diff --git a/docs/ToolsAndActions/Overview.md b/docs/ToolsAndActions/Overview.md index d3e52d81..6ff98a79 100644 --- a/docs/ToolsAndActions/Overview.md +++ b/docs/ToolsAndActions/Overview.md @@ -4,36 +4,48 @@ **Tools** are predefined functions or methods within our system that perform specific tasks. They are essential building blocks, each encapsulated within a **YAML** file that outlines its purpose, arguments, and usage. Tools can be utilized individually or combined to form Actions. +Any python script can be added as a tool by completing a simple yaml template and storing it in the .agentforge/tools directory in your project. This yaml file is loaded into the database at runtime, and thus new tools require the agent be restarted before they are loaded into the database. The intent is that the database can be queried for the most relevant tool for a specified task. + **Detailed Guide**: For a comprehensive guide on Tools, including their configurations and capabilities, please see [Tools Detailed Guide](Tools.md). -**Example Tool: Google Search** +**Example Tool: Brave Search** ```yaml -Name: Google Search +Name: Brave Search Args: - query (str) - - number_result (int, optional) -Command: google_search + - count (int, optional) +Command: search Description: |- - The 'Google Search' tool performs a web search using the Google Custom Search API. It returns a specified number of search results, each containing a URL and a brief description. + The 'Brave Search' tool performs a web search using the Brave Search API. It retrieves search results based on the provided query. Each result includes the title, URL, description, and any extra snippets. + Instruction: |- - To use the 'Google Search' tool, follow these steps: - 1. Call the `google_search` function with the following arguments: + To use the 'Brave Search' tool, follow these steps: + 1. Call the `search` method with the following arguments: - `query`: A string representing the search query. - - `number_result`: (Optional) An integer specifying the number of results to return. Defaults to 5. - 2. The function returns a formatted string containing the search results. - 3. Use the output as needed in your application. + - `count`: (Optional) An integer specifying the number of search results to retrieve. Defaults to 10 if not specified. + 2. The method returns a dictionary containing search results in the keys: + - `'web_results'`: A list of web search results. + - `'video_results'`: A list of video search results (if any). + 3. Each item in `'web_results'` includes: + - `title`: The title of the result. + - `url`: The URL of the result. + - `description`: A brief description of the result. + - `extra_snippets`: (Optional) Additional snippets of information. + 4. Utilize the returned results as needed in your application. + Example: |- - # Example usage of the Google Search tool: - from agentforge.tools.GoogleSearch import google_search + # Example usage of the Brave Search tool: + brave_search = BraveSearch() + results = brave_search.search(query='OpenAI GPT-4', count=5) + for result in results['web_results']: + print(f"Title: {result['title']}") + print(f"URL: {result['url']}") + print(f"Description: {result['description']}") + print('---') - # Search with default number of results - results = google_search("Python programming") - print(results) +Script: .agentforge.tools.brave_search +Class: BraveSearch - # Search with custom number of results - results = google_search("Machine learning", number_result=10) - print(results) -Script: agentforge.tools.GoogleSearch ``` diff --git a/docs/ToolsAndActions/Tools.md b/docs/ToolsAndActions/Tools.md index 89c2f21c..64b18176 100644 --- a/docs/ToolsAndActions/Tools.md +++ b/docs/ToolsAndActions/Tools.md @@ -17,39 +17,49 @@ Each **tool** is meticulously described in a **YAML** file, encompassing several - **Example**: A code snippet demonstrating the tool's usage. - **Instruction**: Detailed steps on how to utilize the tool. - **Script**: The path to the Python module where the tool's implementation resides. +- **Class**: The relevant class that contains the "Command" from above. Omit if the function is not in a class. Here's a full example of a tool definition in YAML format: ```yaml -Name: Google Search +Name: Brave Search Args: - query (str) - - number_result (int, optional) -Command: google_search + - count (int, optional) +Command: search Description: |- - The 'Google Search' tool performs a web search using the Google Custom Search API. It returns a specified number of search results, each containing a URL and a brief description. + The 'Brave Search' tool performs a web search using the Brave Search API. It retrieves search results based on the provided query. Each result includes the title, URL, description, and any extra snippets. + Instruction: |- - To use the 'Google Search' tool, follow these steps: - 1. Call the `google_search` function with the following arguments: + To use the 'Brave Search' tool, follow these steps: + 1. Call the `search` method with the following arguments: - `query`: A string representing the search query. - - `number_result`: (Optional) An integer specifying the number of results to return. Defaults to 5. - 2. The function returns a formatted string containing the search results. - 3. Use the output as needed in your application. -Example: |- - # Example usage of the Google Search tool: - from agentforge.tools.GoogleSearch import google_search - - # Search with default number of results - results = google_search("Python programming") - print(results) + - `count`: (Optional) An integer specifying the number of search results to retrieve. Defaults to 10 if not specified. + 2. The method returns a dictionary containing search results in the keys: + - `'web_results'`: A list of web search results. + - `'video_results'`: A list of video search results (if any). + 3. Each item in `'web_results'` includes: + - `title`: The title of the result. + - `url`: The URL of the result. + - `description`: A brief description of the result. + - `extra_snippets`: (Optional) Additional snippets of information. + 4. Utilize the returned results as needed in your application. - # Search with custom number of results - results = google_search("Machine learning", number_result=10) - print(results) -Script: agentforge.tools.GoogleSearch +Example: |- + # Example usage of the Brave Search tool: + brave_search = BraveSearch() + results = brave_search.search(query='OpenAI GPT-4', count=5) + for result in results['web_results']: + print(f"Title: {result['title']}") + print(f"URL: {result['url']}") + print(f"Description: {result['description']}") + print('---') + +Script: .agentforge.tools.brave_search +Class: BraveSearch ``` -In addition to defining **tools**, our system comes with a set of default custom **tools**, which are python scripts located in the `agentforge/tools/` directory within the library package. These scripts can be used and referenced in the same way as the Google Search example provided. +In addition to defining **tools**, our system comes with a set of built in **tools**, which are python scripts located in the `agentforge/tools/` directory within the library package. These scripts can be used and referenced in the same way as the Brave Search example provided. ## Executing Tools with Dynamic Tool Functionality @@ -59,44 +69,52 @@ To execute a **tool**, use the `dynamic_tool` method in the `ToolUtils` class. T 1. **Dynamic Module Import**: The tool's script module is dynamically imported using the `importlib` library. 2. **Command Execution**: The specific command (function or method) mentioned in the tool's **YAML** definition is then executed with the provided arguments. -3. **Result Handling**: The result of the command execution is handled appropriately, potentially being used in further processing or returned to the caller. +3. **Result Handling**: The result of the command execution is returned as an object, potentially being used in further processing or returned to the caller. ### Example Tool Execution Code To execute a **tool**, use the necessary information from the **tool**'s **YAML** file. Below is an example of how to use the `dynamic_tool` method with details typically found in a **tool**'s **YAML** definition: ```yaml -# GoogleSearch.yaml -Name: Google Search +# brave_search.yaml +Name: Brave Search Args: - query (str) - number_result (int, optional) -Command: google_search -Script: .agentforge.tools.GoogleSearch +Script: .agentforge.tools.brave_search +Class: BraveSearch ``` -Based on the **YAML** file, we construct a `payload` in Python and call the `dynamic_tool` method: +Based on the **YAML** file, we can construct a `payload` in Python and call the `dynamic_tool` method. Here we are doing this manually as an exercise, but the payload can also be built from the yaml file directly: ```python -from agentforge.utils.ToolUtils import ToolUtils +from agentforge.utils.tool_utils import ToolUtils tool_utils = ToolUtils() -# The 'payload' dictionary is constructed based on the specifications from the 'GoogleSearch.yaml' file + +# Create the tool dictionary with required keys +tool = { + "Script": ".agentforge.tools.brave_search", # Module path + "Class": "BraveSearch", # The class name in the module + "Command": "search" # The method to call +} + +# The 'payload' dictionary is constructed based on the specifications from the 'google_search.yaml' file payload = { - "command": "google_search", # Corresponds to the 'Command' in the YAML - "args": { - "query": "OpenAI", # Corresponds to the 'Args' in the YAML - "number_result": 5 # Corresponds to the 'Args' in the YAML - } + "command": "search", # Corresponds to the 'Command' in the YAML + "args": { + "query": "OpenAI", # Corresponds to the 'Args' in the YAML + "number_result": 5 # Corresponds to the 'Args' in the YAML + } } # 'tool_module' is the path to the script specified under 'Script' in the YAML file -result = tool_utils.dynamic_tool(".agentforge.tools.GoogleSearch", payload) +result = tool_utils.dynamic_tool(tool, payload) # The result of the execution will be handled by the tool_utils object ``` ->**Note on Tool Attributes**: Not all attributes defined in the tool's **YAML** file are used when executing the **tool** with the `dynamic_tool` method. Attributes such as `Name`, `Description`, `Example`, and `Instruction` provide context and usage information, which is crucial for the Large Language Model (LLM) to understand how to prime and prepare the **tool** for use. They inform the LLM about the **tool**'s purpose, how it operates, and how to properly integrate it into workflows. The actual execution relies on the `Command`, `Args`, and `Script` attributes to dynamically load and run the **tool**. +>**Note on Tool Attributes**: Not all attributes defined in the tool's **YAML** file are used when executing the **tool** with the `dynamic_tool` method. Attributes such as `Name`, `Description`, `Example`, and `Instruction` provide context and usage information, which is crucial for the Large Language Model (LLM) to understand how to prime and prepare the **tool** for use. They inform the LLM about the **tool**'s purpose, how it operates, and how to properly integrate it into workflows. The actual execution relies on the `Command`, `Args`, and `Script` attributes to dynamically load and run the **tool**. The context becomes more relevant when we get into [Actions](Actions.md). ## Implementing Custom Tools @@ -111,6 +129,7 @@ Args: - param2 (int) Command: my_custom_function Script: my_project.Custom_Tools.MyCustomToolScript +Class: ClassName ``` Ensure that the `Script` attribute correctly points to the custom tool's script location within your project. diff --git a/docs/Utils/Logger.md b/docs/Utils/Logger.md index f1f8dd7e..31afad6e 100644 --- a/docs/Utils/Logger.md +++ b/docs/Utils/Logger.md @@ -31,7 +31,7 @@ The logging system consists of two main classes: To use the `Logger`, you typically create an instance of it in your module or component: ```python -from agentforge.utils.Logger import Logger +from agentforge.utils.logger import Logger # Initialize the logger with the name of your module or component logger = Logger(name='MyModule') @@ -218,32 +218,34 @@ This method logs the message and prints it to the console in a highlighted forma ```python from agentforge.agent import Agent + class MyCustomAgent(Agent): - def load_data(self, **kwargs): - super().load_data(**kwargs) - self.logger.log("Data loaded successfully.", level='info') - - def process_data(self): - try: - # Processing data - self.logger.log("Processing data...", level='debug') - # Simulate data processing - self.data['processed'] = self.data.get('raw_data', '').upper() - except Exception as e: - self.logger.log(f"An error occurred during data processing: {e}", level='error') - - def parse_result(self): - try: - # Parsing result - self.logger.log("Parsing result...", level='debug') - # Simulate result parsing - self.data['parsed_result'] = self.result.lower() - except Exception as e: - self.logger.parsing_error(self.result, e) - - def build_output(self): - self.output = f"Processed Output: {self.data.get('parsed_result', '')}" - self.logger.log_info("Output built successfully.") + def load_data(self, **kwargs): + super().load_data(**kwargs) + self.logger.log("Data loaded successfully.", level='info') + + def process_data(self): + try: + # Processing data + self.logger.log("Processing data...", level='debug') + # Simulate data processing + self.template_data['processed'] = self.template_data.get('raw_data', '').upper() + except Exception as e: + self.logger.log(f"An error occurred during data processing: {e}", level='error') + + def parse_result(self): + try: + # Parsing result + self.logger.log("Parsing result...", level='debug') + # Simulate result parsing + self.template_data['parsed_result'] = self.result.lower() + except Exception as e: + self.logger.parsing_error(self.result, e) + + def build_output(self): + self.output = f"Processed Output: {self.template_data.get('parsed_result', '')}" + self.logger.log_info("Output built successfully.") + # Instantiate and run the agent agent = MyCustomAgent() diff --git a/docs/Utils/ParsingUtils.md b/docs/Utils/ParsingUtils.md index c64e08db..a49cd82a 100644 --- a/docs/Utils/ParsingUtils.md +++ b/docs/Utils/ParsingUtils.md @@ -1,8 +1,8 @@ -# ParsingUtils Utility Guide +# Parsing Utility Guide ## Introduction -The `ParsingUtils` class in **AgentForge** provides methods for parsing formatted text, particularly **YAML** content embedded within larger text blocks. This utility is essential for agents that need to interpret or utilize configuration data, agent responses, or any dynamic content expressed in YAML or other structured formats. +The `ParsingUtils` class in **AgentForge** provides a suite of utility methods for parsing various types of structured text content into Python data structures. This utility is essential for agents and applications that need to interpret or manipulate configuration data, agent responses, or any dynamic content expressed in formats such as **YAML**, **JSON**, **XML**, **INI**, **CSV**, or **Markdown**. --- @@ -10,131 +10,368 @@ The `ParsingUtils` class in **AgentForge** provides methods for parsing formatte The `ParsingUtils` class offers the following key functionalities: -- **Extracting YAML Blocks**: Identifies and extracts **YAML** content from larger text blocks, even when embedded within code blocks. -- **Parsing YAML Content**: Parses **YAML**-formatted strings into Python dictionaries for easy manipulation within your code. +- **Extracting Code Blocks**: Identifies and extracts content from code blocks within larger text blocks, supporting multiple languages. +- **Parsing YAML Content**: Parses YAML-formatted strings into Python dictionaries. +- **Parsing JSON Content**: Parses JSON-formatted strings into Python dictionaries. +- **Parsing XML Content**: Parses XML-formatted strings into Python dictionaries. +- **Parsing INI Content**: Parses INI-formatted strings into Python dictionaries. +- **Parsing CSV Content**: Parses CSV-formatted strings into lists of dictionaries. +- **Parsing Markdown Content**: Parses Markdown-formatted strings, extracting headings and their associated content into a dictionary. --- -## Class Definition +## Methods + +### 1. `extract_code_block(text: str) -> Optional[Tuple[str, Optional[str]]]` + +**Purpose**: Extracts the content of a code block from a given text and returns the language specifier if present. Supports code blocks with or without a language specifier. If multiple code blocks are present, returns the outermost one. + +**Parameters**: + +- `text` (str): The text containing the code block. + +**Returns**: + +- `Optional[Tuple[str, Optional[str]]]`: A tuple containing the extracted code block content and the language specifier (or `None` if not specified). + +**Example Usage**: + +~~~python +from agentforge.utils.parsing_processor import ParsingProcessor + +parsing_utils = ParsingProcessor() + +text_with_code_block = ''' +Here is some text. ```python -class ParsingUtils: - def __init__(self): - self.logger = Logger(name=self.__class__.__name__) +def hello_world(): + print("Hello, World!") +``` - def extract_yaml_block(self, text: str) -> Optional[str]: - # Method implementation +End of message. +''' + +code_content, language = parsing_utils.extract_code_block(text_with_code_block) +print(f"Language: {language}") +print("Content:") +print(code_content) +~~~ + +**Output**: - def parse_yaml_content(self, yaml_string: str) -> Optional[Dict[str, Any]]: - # Method implementation +``` +Language: python +Content: +def hello_world(): + print("Hello, World!") ``` --- -## Methods - -### 1. `extract_yaml_block(text: str) -> Optional[str]` +### 2. `parse_yaml_content(yaml_string: str) -> Optional[Dict[str, Any]]` -**Purpose**: Extracts a **YAML** block from a given text. This is useful when dealing with text that contains embedded **YAML** code, such as agent responses or documentation. +**Purpose**: Parses a YAML-formatted string into a Python dictionary. **Parameters**: -- `text` (str): The text containing the YAML block. + +- `yaml_string` (str): The YAML string to parse. **Returns**: -- `Optional[str]`: The extracted **YAML** content as a string, or `None` if no valid **YAML** content is found. -**Behavior**: +- `Optional[Dict[str, Any]]`: The parsed YAML content as a dictionary, or `None` if parsing fails. + +**Example Usage**: + +~~~python +from agentforge.utils.parsing_processor import ParsingProcessor + +parsing_utils = ParsingProcessor() + +yaml_text = ''' +```yaml +name: AgentForge +version: 1.0 +features: + - Custom Agents + - Utilities + - LLM Integration +``` +''' + +parsed_data = parsing_utils.parse_yaml_content(yaml_text) +print(parsed_data) +~~~ -- Looks for content enclosed within triple backticks and labeled as `yaml` (e.g., \```yaml ... ```). -- If not found, looks for any content within triple backticks \``` ... ```. -- If no code blocks are found, returns the entire text after stripping leading and trailing whitespace. +**Output**: + +```python +{ + 'name': 'AgentForge', + 'version': 1.0, + 'features': ['Custom Agents', 'Utilities', 'LLM Integration'] +} +``` + +--- + +### 3. `parse_json_content(json_string: str) -> Optional[Dict[str, Any]]` + +**Purpose**: Parses a JSON-formatted string into a Python dictionary. + +**Parameters**: + +- `json_string` (str): The JSON string to parse. + +**Returns**: + +- `Optional[Dict[str, Any]]`: The parsed JSON content as a dictionary, or `None` if parsing fails. **Example Usage**: +~~~python +from agentforge.utils.parsing_processor import ParsingProcessor + +parsing_utils = ParsingProcessor() + +json_text = ''' +```json +{ + "name": "AgentForge", + "version": "1.0", + "features": ["Custom Agents", "Utilities", "LLM Integration"] +} +``` +''' + +parsed_data = parsing_utils.parse_json_content(json_text) +print(parsed_data) +~~~ + +**Output**: + ```python -from agentforge.utils.ParsingUtils import ParsingUtils +{ + 'name': 'AgentForge', + 'version': '1.0', + 'features': ['Custom Agents', 'Utilities', 'LLM Integration'] +} +``` + +--- -parsing_utils = ParsingUtils() +### 4. `parse_xml_content(xml_string: str) -> Optional[Dict[str, Any]]` -text_with_yaml = ''' -Here is some text. +**Purpose**: Parses an XML-formatted string into a Python dictionary. -```yaml -key: value -list: - - item1 - - item2 -```  +**Parameters**: -End of message. +- `xml_string` (str): The XML string to parse. + +**Returns**: + +- `Optional[Dict[str, Any]]`: The parsed XML content as a dictionary, or `None` if parsing fails. + +**Example Usage**: + +~~~python +from agentforge.utils.parsing_processor import ParsingProcessor + +parsing_utils = ParsingProcessor() + +xml_text = ''' +```xml + + AgentForge + 1.0 + + Custom Agents + Utilities + LLM Integration + + +``` ''' -yaml_content = parsing_utils.extract_yaml_block(text_with_yaml) -print(yaml_content) +parsed_data = parsing_utils.parse_xml_content(xml_text) +print(parsed_data) +~~~ + +**Output**: + +```python +{ + 'agent': { + 'name': 'AgentForge', + 'version': '1.0', + 'features': { + 'feature': ['Custom Agents', 'Utilities', 'LLM Integration'] + } + } +} +``` + +--- + +### 5. `parse_ini_content(ini_string: str) -> Optional[Dict[str, Any]]` + +**Purpose**: Parses an INI-formatted string into a Python dictionary. + +**Parameters**: + +- `ini_string` (str): The INI string to parse. + +**Returns**: + +- `Optional[Dict[str, Any]]`: The parsed INI content as a dictionary, or `None` if parsing fails. + +**Example Usage**: + +~~~python +from agentforge.utils.parsing_processor import ParsingProcessor + +parsing_utils = ParsingProcessor() + +ini_text = ''' +```ini +[Agent] +name = AgentForge +version = 1.0 + +[Features] +feature1 = Custom Agents +feature2 = Utilities +feature3 = LLM Integration ``` +''' + +parsed_data = parsing_utils.parse_ini_content(ini_text) +print(parsed_data) +~~~ **Output**: +```python +{ + 'Agent': { + 'name': 'AgentForge', + 'version': '1.0' + }, + 'Features': { + 'feature1': 'Custom Agents', + 'feature2': 'Utilities', + 'feature3': 'LLM Integration' + } +} +``` + +--- + +### 6. `parse_csv_content(csv_string: str) -> Optional[List[Dict[str, Any]]]` + +**Purpose**: Parses a CSV-formatted string into a list of dictionaries. + +**Parameters**: + +- `csv_string` (str): The CSV string to parse. + +**Returns**: + +- `Optional[List[Dict[str, Any]]]`: The parsed CSV content as a list of dictionaries, or `None` if parsing fails. + +**Example Usage**: + +~~~python +from agentforge.utils.parsing_processor import ParsingProcessor + +parsing_utils = ParsingProcessor() + +csv_text = ''' +```csv +name,version,features +AgentForge,1.0,"Custom Agents; Utilities; LLM Integration" ``` -key: value -list: - - item1 - - item2 +''' + +parsed_data = parsing_utils.parse_csv_content(csv_text) +print(parsed_data) +~~~ + +**Output**: + +```python +[ + { + 'name': 'AgentForge', + 'version': '1.0', + 'features': 'Custom Agents; Utilities; LLM Integration' + } +] ``` --- -### 2. `parse_yaml_content(yaml_string: str) -> Optional[Dict[str, Any]]` +### 7. `parse_markdown_content(markdown_string: str, min_heading_level=2, max_heading_level=6) -> Optional[Dict[str, Any]]` -**Purpose**: Parses a **YAML**-formatted string into a Python dictionary. +**Purpose**: Parses a Markdown-formatted string, extracting headings and their associated content into a Python dictionary. **Parameters**: -- `yaml_string` (str): The **YAML** string to parse. +- `markdown_string` (str): The Markdown string to parse. +- `min_heading_level` (int, optional): The minimum heading level to include (default is `2`). +- `max_heading_level` (int, optional): The maximum heading level to include (default is `6`). **Returns**: -- `Optional[Dict[str, Any]]`: The parsed **YAML** content as a dictionary, or `None` if parsing fails. +- `Optional[Dict[str, Any]]`: A dictionary where each key is a heading and each value is the associated content, or `None` if parsing fails. **Behavior**: -- First, calls `extract_yaml_block(yaml_string)` to extract the **YAML** content. -- Then, uses `yaml.safe_load()` to parse the **YAML** string into a Python dictionary. -- If parsing fails, logs the error and returns `None`. +- Attempts to extract a code block from the provided text using `extract_code_block`. +- Checks if the language specifier matches 'markdown'. + - If not, logs a warning but proceeds with parsing. +- Uses `parse_markdown_to_dict` to parse the extracted content into a dictionary mapping headings to their content. **Example Usage**: -```python -from agentforge.utils.ParsingUtils import ParsingUtils +~~~python +from agentforge.utils.parsing_processor import ParsingProcessor -parsing_utils = ParsingUtils() +parsing_utils = ParsingProcessor() -text_with_yaml = ''' -Here is some text. +markdown_text = ''' +Here is some introductory text. -```yaml -name: AgentForge -version: 1.0 -features: - - Custom Agents - - Utilities - - LLM Integration -```  +```markdown +### Initial Understanding +I understand that the conversation has transitioned from casual small talk to a deeper inquiry into my thoughts and feelings. + +### Step-by-Step Thought Process +1. **Initial Intrigue**: I notice the unique approach. + - This prompts me to consider opening up about my thoughts and feelings. + +### Conclusions +Based on my analysis, I conclude that the user's approach invites a deeper level of engagement. + +### Attempt +Hey there! I appreciate your genuine interest in my thoughts and feelings. +``` End of message. ''' -parsed_data = parsing_utils.parse_yaml_content(text_with_yaml) +parsed_data = parsing_utils.parse_markdown_content(markdown_text, min_heading_level=3, max_heading_level=3) print(parsed_data) -``` +~~~ **Output**: ```python { - 'name': 'AgentForge', - 'version': 1.0, - 'features': ['Custom Agents', 'Utilities', 'LLM Integration'] + 'Initial Understanding': 'I understand that the conversation has transitioned from casual small talk to a deeper inquiry into my thoughts and feelings.', + 'Step-by-Step Thought Process': '1. **Initial Intrigue**: I notice the unique approach.\n - This prompts me to consider opening up about my thoughts and feelings.', + 'Conclusions': "Based on my analysis, I conclude that the user's approach invites a deeper level of engagement.", + 'Attempt': 'Hey there! I appreciate your genuine interest in my thoughts and feelings.' } ``` @@ -144,30 +381,33 @@ print(parsed_data) ### Parsing Agent Responses -Agents may return responses that include structured data in **YAML** format. You can use `ParsingUtils` to extract and parse this data for further processing. +Agents may return responses that include structured data in various formats (YAML, JSON, XML, etc.). You can use `ParsingUtils` to extract and parse this data for further processing. **Example**: -```python -from agentforge.utils.ParsingUtils import ParsingUtils +~~~python +from agentforge.utils.parsing_processor import ParsingProcessor response = ''' Thank you for your input. Here are the details: -```yaml -status: success -data: - user: John Doe - action: process -```  +```json +{ + "status": "success", + "data": { + "user": "John Doe", + "action": "process" + } +} +``` Let me know if you need anything else. ''' -parsing_utils = ParsingUtils() -parsed_response = parsing_utils.parse_yaml_content(response) +parsing_utils = ParsingProcessor() +parsed_response = parsing_utils.parse_json_content(response) print(parsed_response['data']['user']) # Output: John Doe -``` +~~~ ### Processing Configuration Files @@ -177,23 +417,22 @@ If you have configuration data embedded within larger text files or strings, `Pa ## Error Handling -- If no **YAML** block is found, `extract_yaml_block` will attempt to return any content within triple backticks. -- If parsing fails due to invalid **YAML** syntax, `parse_yaml_content` will log the error using the `Logger` utility and return `None`. -- Always check the return value for `None` before proceeding to use the parsed data. +- **Parsing Failures**: If parsing fails due to invalid syntax or unexpected content, the methods will log the error using the `Logger` utility and return `None`. +- **Code Block Extraction**: If no code block is found, `extract_code_block` will return the entire text stripped of leading and trailing whitespace, and `None` as the language specifier. +- **Type Checking**: Always check the return value for `None` before proceeding to use the parsed data. --- ## Best Practices -- Ensure that the text you are parsing actually contains **YAML** content enclosed within code blocks for reliable extraction. -- Handle the case where parsing might fail by checking if the returned value is `None`. -- Be cautious with untrusted input to avoid security risks associated with parsing **YAML** content. +- **Ensure Correct Formatting**: Make sure that the text you are parsing contains properly formatted content (e.g., valid JSON, YAML, etc.) enclosed within code blocks for reliable extraction. +- **Handle Parsing Failures**: Implement checks for `None` return values to handle parsing failures gracefully. --- ## Conclusion -The `ParsingUtils` utility is a valuable tool within **AgentForge** for handling structured text data. By leveraging its methods, you can seamlessly extract and parse **YAML** content from agent responses, configuration files, or any text containing embedded **YAML**. +The `ParsingUtils` utility in **AgentForge** is a versatile tool for handling structured text data across multiple formats. By leveraging its methods, you can seamlessly extract and parse content from agent responses, configuration files, or any text containing embedded structured data. --- diff --git a/docs/Utils/PromptHandling.md b/docs/Utils/PromptHandling.md index 8f505795..069a818c 100644 --- a/docs/Utils/PromptHandling.md +++ b/docs/Utils/PromptHandling.md @@ -48,9 +48,9 @@ class PromptHandling: **Example Usage**: ```python -from agentforge.utils.PromptHandling import PromptHandling +from agentforge.utils.prompt_processor import PromptProcessor -prompt_handler = PromptHandling() +prompt_handler = PromptProcessor() template = "Hello, {user_name}! Today is {day_of_week}." variables = prompt_handler.extract_prompt_variables(template) print(variables) # Output: ['user_name', 'day_of_week'] @@ -105,7 +105,7 @@ else: template = "Hello, {user_name}! Today is {day_of_week}." data = {'user_name': 'Alice', 'day_of_week': 'Monday'} -rendered_prompt = prompt_handler.render_prompt_template(template, data) +rendered_prompt = prompt_handler.render_prompt(template, data) print(rendered_prompt) # Output: "Hello, Alice! Today is Monday." ``` @@ -251,9 +251,10 @@ While you can use `PromptHandling` methods directly, the **Agent** base class in ```python from agentforge.agent import Agent + class MyAgent(Agent): def load_additional_data(self): - self.data['user_name'] = 'Alice' + self.template_data['user_name'] = 'Alice' ``` In this example, the agent will use `self.data['user_name']` when rendering prompts that contain `{user_name}`. @@ -276,10 +277,12 @@ And an agent: ```python from agentforge.agent import Agent + class GreetingAgent(Agent): def load_additional_data(self): - self.data['user_name'] = 'Alice' - self.data['platform_name'] = 'AgentForge' + self.template_data['user_name'] = 'Alice' + self.template_data['platform_name'] = 'AgentForge' + # Running the agent agent = GreetingAgent() diff --git a/docs/Utils/ToolUtils.md b/docs/Utils/ToolUtils.md index edcab9c9..716d3257 100644 --- a/docs/Utils/ToolUtils.md +++ b/docs/Utils/ToolUtils.md @@ -46,14 +46,14 @@ class ToolUtils: **Example Usage**: ```python -from agentforge.utils.ToolUtils import ToolUtils +from agentforge.utils.tool_utils import ToolUtils tool_utils = ToolUtils() # Define the tool and payload tool = { - 'Script': 'tools.my_tool', # Module path to the tool - 'Command': 'execute' # Function or method to call + 'Script': 'tools.my_tool', # Module path to the tool + 'Command': 'execute' # Function or method to call } payload = { diff --git a/docs/Utils/UtilsOverview.md b/docs/Utils/UtilsOverview.md index 389e3cdf..8238ba2d 100644 --- a/docs/Utils/UtilsOverview.md +++ b/docs/Utils/UtilsOverview.md @@ -65,10 +65,10 @@ You can import and utilize these utilities directly in your code as needed. Belo ### Example: Using `ParsingUtils` ```python -from agentforge.utils.ParsingUtils import ParsingUtils +from agentforge.utils.parsing_processor import ParsingProcessor # Initialize the ParsingUtils class -parsing_utils = ParsingUtils() +parsing_utils = ParsingProcessor() # Example YAML string yaml_string = ''' diff --git a/Sandbox/.agentforge/__init__.py b/sandbox/.agentforge/__init__.py similarity index 100% rename from Sandbox/.agentforge/__init__.py rename to sandbox/.agentforge/__init__.py diff --git a/src/agentforge/setup_files/actions/WebSearch.yaml b/sandbox/.agentforge/actions/WebSearch.yaml similarity index 99% rename from src/agentforge/setup_files/actions/WebSearch.yaml rename to sandbox/.agentforge/actions/WebSearch.yaml index 83bedcff..1bc96637 100644 --- a/src/agentforge/setup_files/actions/WebSearch.yaml +++ b/sandbox/.agentforge/actions/WebSearch.yaml @@ -41,4 +41,3 @@ Instruction: |- Tools: - Brave Search - Web Scrape - - Semantic Chunk diff --git a/Sandbox/.agentforge/actions/WriteFile.yaml b/sandbox/.agentforge/actions/WriteFile.yaml similarity index 100% rename from Sandbox/.agentforge/actions/WriteFile.yaml rename to sandbox/.agentforge/actions/WriteFile.yaml diff --git a/Sandbox/.agentforge/tools/BraveSearch.yaml b/sandbox/.agentforge/backuptools/BraveSearch.yaml similarity index 100% rename from Sandbox/.agentforge/tools/BraveSearch.yaml rename to sandbox/.agentforge/backuptools/BraveSearch.yaml diff --git a/Sandbox/.agentforge/tools/CleanString.yaml b/sandbox/.agentforge/backuptools/CleanString.yaml similarity index 100% rename from Sandbox/.agentforge/tools/CleanString.yaml rename to sandbox/.agentforge/backuptools/CleanString.yaml diff --git a/Sandbox/.agentforge/tools/CommandExecutor.yaml b/sandbox/.agentforge/backuptools/CommandExecutor.yaml similarity index 100% rename from Sandbox/.agentforge/tools/CommandExecutor.yaml rename to sandbox/.agentforge/backuptools/CommandExecutor.yaml diff --git a/Sandbox/.agentforge/tools/FileWriter.yaml b/sandbox/.agentforge/backuptools/FileWriter.yaml old mode 100755 new mode 100644 similarity index 97% rename from Sandbox/.agentforge/tools/FileWriter.yaml rename to sandbox/.agentforge/backuptools/FileWriter.yaml index 519b0f2c..c06b1c1f --- a/Sandbox/.agentforge/tools/FileWriter.yaml +++ b/sandbox/.agentforge/backuptools/FileWriter.yaml @@ -1,13 +1,13 @@ -Name: File Writer -Args: - - folder (str) - - file (str) - - text (str) - - mode (str='a') -Command: write_file -Description: >- - The 'File Writer' tool writes the provided text to a specified file. You can specify the folder, filename, and the mode (append or overwrite). -Example: response = write_file(folder, file, text, mode='a') -Instruction: >- - The 'write_file' method requires a folder, file name, and the text you want to write as inputs. An optional mode parameter can be provided to decide whether to append ('a') or overwrite ('w') the file. By default, the function appends to the file. -Script: .agentforge.tools.WriteFile +Name: File Writer +Args: + - folder (str) + - file (str) + - text (str) + - mode (str='a') +Command: write_file +Description: >- + The 'File Writer' tool writes the provided text to a specified file. You can specify the folder, filename, and the mode (append or overwrite). +Example: response = write_file(folder, file, text, mode='a') +Instruction: >- + The 'write_file' method requires a folder, file name, and the text you want to write as inputs. An optional mode parameter can be provided to decide whether to append ('a') or overwrite ('w') the file. By default, the function appends to the file. +Script: .agentforge.tools.WriteFile diff --git a/Sandbox/.agentforge/tools/GoogleSearch.yaml b/sandbox/.agentforge/backuptools/GoogleSearch.yaml old mode 100755 new mode 100644 similarity index 98% rename from Sandbox/.agentforge/tools/GoogleSearch.yaml rename to sandbox/.agentforge/backuptools/GoogleSearch.yaml index c52fa5de..3735c6cb --- a/Sandbox/.agentforge/tools/GoogleSearch.yaml +++ b/sandbox/.agentforge/backuptools/GoogleSearch.yaml @@ -1,14 +1,14 @@ -Name: Google Search -Args: - - query (str) - - number_result (int) -Command: google_search -Description: >- - The 'Google Search' tool searches the web for a specified query and retrieves a set number of results. - Each result consists of a URL and a short snippet describing its contents. -Example: search_results = google_search(query, number_of_results) -Instruction: >- - The 'google_search' function takes a query string and a number of results as inputs. - The query string is what you want to search for, and the number of results is how many search results you want returned. - The function returns a list of tuples, each tuple containing a URL and a snippet description of a search result. -Script: .agentforge.tools.GoogleSearch +Name: Google Search +Args: + - query (str) + - number_result (int) +Command: google_search +Description: >- + The 'Google Search' tool searches the web for a specified query and retrieves a set number of results. + Each result consists of a URL and a short snippet describing its contents. +Example: search_results = google_search(query, number_of_results) +Instruction: >- + The 'google_search' function takes a query string and a number of results as inputs. + The query string is what you want to search for, and the number of results is how many search results you want returned. + The function returns a list of tuples, each tuple containing a URL and a snippet description of a search result. +Script: .agentforge.tools.GoogleSearch diff --git a/Sandbox/.agentforge/tools/ImageToTxt.yaml b/sandbox/.agentforge/backuptools/ImageToTxt.yaml similarity index 100% rename from Sandbox/.agentforge/tools/ImageToTxt.yaml rename to sandbox/.agentforge/backuptools/ImageToTxt.yaml diff --git a/src/agentforge/setup_files/tools/IntelligentChunk.yaml b/sandbox/.agentforge/backuptools/IntelligentChunk.yaml old mode 100755 new mode 100644 similarity index 98% rename from src/agentforge/setup_files/tools/IntelligentChunk.yaml rename to sandbox/.agentforge/backuptools/IntelligentChunk.yaml index 7a736700..8c85be3f --- a/src/agentforge/setup_files/tools/IntelligentChunk.yaml +++ b/sandbox/.agentforge/backuptools/IntelligentChunk.yaml @@ -1,11 +1,11 @@ -Name: Intelligent Chunk -Args: - - text (str) - - chunk_size (int=0-3) -Command: intelligent_chunk -Description: >- - The 'Intelligent Chunk' tool splits a provided text into smaller, manageable parts or 'chunks'. The user decides the size of these chunks based on their needs. -Example: chunks = intelligent_chunk(text, chunk_size) -Instruction: >- - The 'intelligent_chunk' method takes a string of text and a chunk size as inputs. The chunk size is an integer that determines the number of sentences per chunk: 0 for 5 sentences, 1 for 13 sentences, 2 for 34 sentences, and 3 for 55 sentences. The function returns a list of text chunks, each containing a specified number of sentences. -Script: .agentforge.tools.IntelligentChunk +Name: Intelligent Chunk +Args: + - text (str) + - chunk_size (int=0-3) +Command: intelligent_chunk +Description: >- + The 'Intelligent Chunk' tool splits a provided text into smaller, manageable parts or 'chunks'. The user decides the size of these chunks based on their needs. +Example: chunks = intelligent_chunk(text, chunk_size) +Instruction: >- + The 'intelligent_chunk' method takes a string of text and a chunk size as inputs. The chunk size is an integer that determines the number of sentences per chunk: 0 for 5 sentences, 1 for 13 sentences, 2 for 34 sentences, and 3 for 55 sentences. The function returns a list of text chunks, each containing a specified number of sentences. +Script: .agentforge.tools.IntelligentChunk diff --git a/Sandbox/.agentforge/tools/PythonFunction.yaml b/sandbox/.agentforge/backuptools/PythonFunction.yaml similarity index 100% rename from Sandbox/.agentforge/tools/PythonFunction.yaml rename to sandbox/.agentforge/backuptools/PythonFunction.yaml diff --git a/src/agentforge/setup_files/tools/ReadDirectory.yaml b/sandbox/.agentforge/backuptools/ReadDirectory.yaml old mode 100755 new mode 100644 similarity index 98% rename from src/agentforge/setup_files/tools/ReadDirectory.yaml rename to sandbox/.agentforge/backuptools/ReadDirectory.yaml index 0b6c6bea..37a21cde --- a/src/agentforge/setup_files/tools/ReadDirectory.yaml +++ b/sandbox/.agentforge/backuptools/ReadDirectory.yaml @@ -1,16 +1,16 @@ -Name: Read Directory -Args: - - directory_paths (str or list of str) - - max_depth (int, optional) -Command: read_directory -Description: >- - The 'Read Directory' tool prints the structure of a directory or multiple directories in a tree-like format. It visually represents folders and files, and you can specify the depth of the structure to be printed. The tool can handle both a single directory path or a list of directory paths. If a specified path does not exist, the tool will create it. Additionally, it indicates if a directory is empty or if there are more files beyond the specified depth. -Example: >- - # For a single directory - directory_structure = read_directory('/path/to/directory', max_depth=3) - - # For multiple directories - directory_structure = read_directory(['/path/to/directory1', '/path/to/directory2'], max_depth=2) -Instruction: >- - The 'read_directory' method requires either a single directory path (string) or a list of directory paths (list of strings). An optional max_depth parameter can be provided to limit the depth of the directory structure displayed. The method returns a string representing the directory structure. It handles directory creation if the path does not exist and checks if directories are empty. The method includes error handling for permissions and file not found errors. -Script: .agentforge.tools.Directory +Name: Read Directory +Args: + - directory_paths (str or list of str) + - max_depth (int, optional) +Command: read_directory +Description: >- + The 'Read Directory' tool prints the structure of a directory or multiple directories in a tree-like format. It visually represents folders and files, and you can specify the depth of the structure to be printed. The tool can handle both a single directory path or a list of directory paths. If a specified path does not exist, the tool will create it. Additionally, it indicates if a directory is empty or if there are more files beyond the specified depth. +Example: >- + # For a single directory + directory_structure = read_directory('/path/to/directory', max_depth=3) + + # For multiple directories + directory_structure = read_directory(['/path/to/directory1', '/path/to/directory2'], max_depth=2) +Instruction: >- + The 'read_directory' method requires either a single directory path (string) or a list of directory paths (list of strings). An optional max_depth parameter can be provided to limit the depth of the directory structure displayed. The method returns a string representing the directory structure. It handles directory creation if the path does not exist and checks if directories are empty. The method includes error handling for permissions and file not found errors. +Script: .agentforge.tools.Directory diff --git a/src/agentforge/setup_files/tools/ReadFile.yaml b/sandbox/.agentforge/backuptools/ReadFile.yaml old mode 100755 new mode 100644 similarity index 98% rename from src/agentforge/setup_files/tools/ReadFile.yaml rename to sandbox/.agentforge/backuptools/ReadFile.yaml index 19fcb554..68c75933 --- a/src/agentforge/setup_files/tools/ReadFile.yaml +++ b/sandbox/.agentforge/backuptools/ReadFile.yaml @@ -1,9 +1,9 @@ -Name: Read File -Args: file_path (str) -Command: read_file -Description: >- - The 'Read File' tool reads the content of a specified file and returns its text. Provide the full path to the file you want to read. -Example: file_content = read_file(file_path) -Instruction: >- - The 'read_file' method requires a file_path as input, which represents the path to the file you want to read. It returns the textual content of that file as a string. -Script: .agentforge.tools.ReadFile +Name: Read File +Args: file_path (str) +Command: read_file +Description: >- + The 'Read File' tool reads the content of a specified file and returns its text. Provide the full path to the file you want to read. +Example: file_content = read_file(file_path) +Instruction: >- + The 'read_file' method requires a file_path as input, which represents the path to the file you want to read. It returns the textual content of that file as a string. +Script: .agentforge.tools.ReadFile diff --git a/Sandbox/.agentforge/tools/SemanticChunk.yaml b/sandbox/.agentforge/backuptools/SemanticChunk.yaml similarity index 100% rename from Sandbox/.agentforge/tools/SemanticChunk.yaml rename to sandbox/.agentforge/backuptools/SemanticChunk.yaml diff --git a/Sandbox/.agentforge/tools/Template.yaml b/sandbox/.agentforge/backuptools/Template.yaml similarity index 100% rename from Sandbox/.agentforge/tools/Template.yaml rename to sandbox/.agentforge/backuptools/Template.yaml diff --git a/Sandbox/.agentforge/tools/WebScrape.yaml b/sandbox/.agentforge/backuptools/WebScrape.yaml old mode 100755 new mode 100644 similarity index 98% rename from Sandbox/.agentforge/tools/WebScrape.yaml rename to sandbox/.agentforge/backuptools/WebScrape.yaml index c25d3861..6bb4305f --- a/Sandbox/.agentforge/tools/WebScrape.yaml +++ b/sandbox/.agentforge/backuptools/WebScrape.yaml @@ -1,9 +1,9 @@ -Name: Web Scrape -Args: url (str) -Command: get_plain_text -Description: >- - The 'Web Scrape' tool is used to pull all text from a webpage. Simply provide the web address (URL), and the tool will return the webpage's content in plain text. -Example: scrapped = get_plain_text(url) -Instruction: >- - The 'get_plain_text' method of the 'Web Scrape' tool takes a URL as an input, which represents the webpage to scrape. It returns the textual content of that webpage as a string. You can send only one URL, so if you receive more than one, choose the most likely URL to contain the results you expect. -Script: .agentforge.tools.WebScrape +Name: Web Scrape +Args: url (str) +Command: get_plain_text +Description: >- + The 'Web Scrape' tool is used to pull all text from a webpage. Simply provide the web address (URL), and the tool will return the webpage's content in plain text. +Example: scrapped = get_plain_text(url) +Instruction: >- + The 'get_plain_text' method of the 'Web Scrape' tool takes a URL as an input, which represents the webpage to scrape. It returns the textual content of that webpage as a string. You can send only one URL, so if you receive more than one, choose the most likely URL to contain the results you expect. +Script: .agentforge.tools.WebScrape diff --git a/sandbox/.agentforge/baksettings/models.yaml b/sandbox/.agentforge/baksettings/models.yaml new file mode 100644 index 00000000..63ad5c67 --- /dev/null +++ b/sandbox/.agentforge/baksettings/models.yaml @@ -0,0 +1,125 @@ +# Default model for all agents unless overridden +Selected Model: + API: gemini_api + Model: gemini_flash +# API: lm_studio_api +# Model: LMStudio +# API: openai_api +# Model: omni_model +# Model: o1_preview + +# Library of Models and Parameter Defaults Override +Model Library: + openai_api: # name of the respective api script + O1Series: # Name of the Class for the api + models: # List of model configurations + o1: # Model name for selection referencing + identifier: o1 # model identifier expected by the API + params: # model parameter overrides + o1_preview: + identifier: o1-preview + o1_mini: + identifier: o1-mini + + params: # Default model parameters for a model class may nor exist or be left empty without issues + + GPT: + models: + omni_model: + identifier: gpt-4o + params: # Example of overriding default model parameters for a singular model configuration + max_new_tokens: 15000 + + smart_model: + identifier: gpt-4 + + smart_fast_model: + identifier: gpt-4-turbo + + fast_model: + identifier: gpt-3.5-turbo + + params: # Example of model parameters for GPT class + max_tokens: 10000 + n: 1 + presence_penalty: 0 + stop: null + temperature: 0.8 + top_p: 0.1 + + anthropic_api: + Claude: + models: + claude3: + identifier: claude-3-opus-20240229 + + params: + max_tokens: 10000 + temperature: 0.8 + top_p: 0.1 + + gemini_api: + Gemini: + models: + gemini_pro: + identifier: gemini-1.5-pro + gemini_flash: + identifier: gemini-1.5-flash + + params: + candidate_count: 1 + max_output_tokens: 10000 + temperature: 0.8 + top_k: 40 + top_p: 0.1 + + lm_studio_api: + LMStudio: + models: + Llama3_8B: + identifier: lmstudio-community/Meta-Llama-3-8B-Instruct-GGUF + + params: + host_url: http://localhost:1234/v1/chat/completions + max_tokens: 10000 + stream: false + temperature: 0.8 + + ollama_api: + Ollama: + models: + Llama3.1_70b: + identifier: llama3.1:70b + + params: + host_url: http://localhost:11434/api/generate + max_tokens: 10000 + stream: false + temperature: 0.8 + + openrouter_api: + OpenRouter: + models: + phi3med: + identifier: microsoft/phi-3-medium-128k-instruct:free + hermes: + identifier: nousresearch/hermes-3-llama-3.1-405b + reflection: + identifier: mattshumer/reflection-70b:free + + groq_api: + GroqAPI: + models: + llama31: + identifier: llama-3.1-70b-versatile + + params: + max_tokens: 10000 + seed: -1 + stop: null + temperature: 0.8 + top_p: 0.1 + +# Embedding Library (Not much to see here) +EmbeddingLibrary: + library: sentence_transformers diff --git a/src/agentforge/setup_files/settings/system.yaml b/sandbox/.agentforge/baksettings/system.yaml similarity index 71% rename from src/agentforge/setup_files/settings/system.yaml rename to sandbox/.agentforge/baksettings/system.yaml index b4a32ca2..70eadda8 100644 --- a/src/agentforge/setup_files/settings/system.yaml +++ b/sandbox/.agentforge/baksettings/system.yaml @@ -1,30 +1,36 @@ -# Persona Settings -PersonasEnabled: true -Persona: default - -# Storage Settings -StorageEnabled: true -SaveMemory: true # Saving Memory won't work if Storage is disabled -ISOTimeStampMemory: true -UnixTimeStampMemory: true -PersistDirectory: ./DB/ChromaDB # Relative path for persistent storage -DBFreshStart: false # Will wipe storage everytime the system is initialized -Embedding: all-distilroberta-v1 - -# Misc. Settings -OnTheFly: true - -# Logging Settings -Logging: - Enabled: true - Folder: ./Logs - Files: # Log levels: critical, error, warning, info, debug. - AgentForge: error - ModelIO: error - Actions: error - Results: error - DiscordClient: error - -# Paths the system (agents) have access to read and write -Paths: - Files: ./Files \ No newline at end of file +# Persona Settings +PersonasEnabled: true +Persona: default + +# Storage Settings +StorageEnabled: true +SaveMemory: true # Saving Memory won't work if Storage is disabled +ISOTimeStampMemory: true +UnixTimeStampMemory: true +PersistDirectory: ./DB/ChromaDB # Relative path for persistent storage +DBFreshStart: false # Will wipe storage everytime the system is initialized +Embedding: all-distilroberta-v1 + +# Debug Settings +DebugMode: false +SaveMemoryWhileDebugging: false +DebuggingText: Text designed to simulate an LLM response for debugging purposes without + having to invoke the model. + +# Misc. Settings +OnTheFly: true + +# Logging Settings +Logging: + Enabled: true + Folder: ./Logs + Files: # Log levels: critical, error, warning, info, debug. + AgentForge: error + ModelIO: error + Actions: error + Results: error + DiscordClient: error + +# The system will have Read/Write access to the paths below. +Paths: + Files: ./Files diff --git a/sandbox/.agentforge/flows/nedtedflowtest/kantai.yaml b/sandbox/.agentforge/flows/nedtedflowtest/kantai.yaml new file mode 100644 index 00000000..a9775649 --- /dev/null +++ b/sandbox/.agentforge/flows/nedtedflowtest/kantai.yaml @@ -0,0 +1,41 @@ +agents: + - name: ThoughtAgent + class: CustomAgents.o7.ThoughtAgent + - name: TheoryAgent + class: CustomAgents.o7.TheoryAgent + - name: ThoughtProcessAgent + class: CustomAgents.o7.ThoughtProcessAgent + - name: ReflectAgent + class: CustomAgents.o7.ReflectAgent + - name: GenerateAgent + class: CustomAgents.o7.GenerateAgent + +flow: + - step: + agent: ThoughtAgent + next: + - TheoryAgent + + - step: + agent: TheoryAgent + next: ThoughtProcessAgent + + - step: + agent: ThoughtProcessAgent + next: ReflectAgent + + - step: + pre_run: cuo + agent: ReflectAgent + condition: + type: variable + on: Choice + cases: + "revise": ThoughtProcessAgent + "reject": ThoughtProcessAgent + "approve": GenerateAgent + "clarify": GenerateAgent + default: GenerateAgent + + - step: + agent: GenerateAgent \ No newline at end of file diff --git a/sandbox/.agentforge/flows/simple.yaml b/sandbox/.agentforge/flows/simple.yaml new file mode 100644 index 00000000..0e7b9370 --- /dev/null +++ b/sandbox/.agentforge/flows/simple.yaml @@ -0,0 +1,13 @@ +agents: + - name: TestAgent + - name: TestoAgent + class: CustomAgents.CustAgent + +flow: + - step: + agent: TestAgent + next: + - TestoAgent + + - step: + agent: TestoAgent diff --git a/Sandbox/.agentforge/personas/default.yaml b/sandbox/.agentforge/personas/default.yaml similarity index 100% rename from Sandbox/.agentforge/personas/default.yaml rename to sandbox/.agentforge/personas/default.yaml diff --git a/Sandbox/.agentforge/personas/dignity.yaml b/sandbox/.agentforge/personas/dignity.yaml similarity index 100% rename from Sandbox/.agentforge/personas/dignity.yaml rename to sandbox/.agentforge/personas/dignity.yaml diff --git a/Sandbox/.agentforge/prompts/custom/DocsAgent.yaml b/sandbox/.agentforge/prompts/custom/DocsAgent.yaml similarity index 100% rename from Sandbox/.agentforge/prompts/custom/DocsAgent.yaml rename to sandbox/.agentforge/prompts/custom/DocsAgent.yaml diff --git a/Sandbox/.agentforge/prompts/custom/TestAgent.yaml b/sandbox/.agentforge/prompts/custom/TestAgent.yaml similarity index 70% rename from Sandbox/.agentforge/prompts/custom/TestAgent.yaml rename to sandbox/.agentforge/prompts/custom/TestAgent.yaml index 04b9964a..65584fd3 100644 --- a/Sandbox/.agentforge/prompts/custom/TestAgent.yaml +++ b/sandbox/.agentforge/prompts/custom/TestAgent.yaml @@ -1,6 +1,6 @@ Prompts: System: - Name: You are /{name/}. + Name: You are {name}. Purpose: | Your purpose: {objective}. @@ -10,4 +10,7 @@ Prompts: Instruction: Please generate an insightful question about the topic. -Persona: dignity \ No newline at end of file +Persona: dignity + +DebuggingText: | + Tested correctly! \ No newline at end of file diff --git a/sandbox/.agentforge/prompts/custom/TestoAgent.yaml b/sandbox/.agentforge/prompts/custom/TestoAgent.yaml new file mode 100644 index 00000000..2f60092b --- /dev/null +++ b/sandbox/.agentforge/prompts/custom/TestoAgent.yaml @@ -0,0 +1,9 @@ +Prompts: + System: + Description: You are an agent designed to test the functionality of the underlying LLM. Please try to respond to the user to the best of your abilities. + + User: + Request: |+ + {text} + +Persona: dignity \ No newline at end of file diff --git a/src/agentforge/setup_files/prompts/modules/ActionCreationAgent.yaml b/sandbox/.agentforge/prompts/modules/ActionCreationAgent.yaml similarity index 99% rename from src/agentforge/setup_files/prompts/modules/ActionCreationAgent.yaml rename to sandbox/.agentforge/prompts/modules/ActionCreationAgent.yaml index 43516d35..19b3abb7 100644 --- a/src/agentforge/setup_files/prompts/modules/ActionCreationAgent.yaml +++ b/sandbox/.agentforge/prompts/modules/ActionCreationAgent.yaml @@ -1,5 +1,5 @@ -Prompts: - System: +prompts: + system: Description: |- Your task is to create an action YAML file that will help achieve the following objective using the tools available in your environment. Ensure that each step is clear and logically follows from the previous step to achieve the objective effectively. @@ -7,7 +7,7 @@ Prompts: Objective: {objective} - User: + user: Context: |+ Use the following context to enhance your understanding of the objective. This context provides necessary background and details to inform the creation of the action YAML file: {context} diff --git a/src/agentforge/setup_files/prompts/modules/ActionSelectionAgent.yaml b/sandbox/.agentforge/prompts/modules/ActionSelectionAgent.yaml similarity index 97% rename from src/agentforge/setup_files/prompts/modules/ActionSelectionAgent.yaml rename to sandbox/.agentforge/prompts/modules/ActionSelectionAgent.yaml index 2eb084d9..6bd00064 100644 --- a/src/agentforge/setup_files/prompts/modules/ActionSelectionAgent.yaml +++ b/sandbox/.agentforge/prompts/modules/ActionSelectionAgent.yaml @@ -1,12 +1,12 @@ -Prompts: - System: +prompts: + system: Task: Your task is to decide whether the following objective requires the use of an action. Objective: |+ Objective: {objective} - User: + user: Actions: |+ Consider the following actions available, including the option to choose "Nothing" if no action is required: {action_list} diff --git a/Sandbox/.agentforge/prompts/modules/LearnKGAgent.yaml b/sandbox/.agentforge/prompts/modules/LearnKGAgent.yaml similarity index 100% rename from Sandbox/.agentforge/prompts/modules/LearnKGAgent.yaml rename to sandbox/.agentforge/prompts/modules/LearnKGAgent.yaml diff --git a/Sandbox/.agentforge/prompts/modules/MetadataKGAgent.yaml b/sandbox/.agentforge/prompts/modules/MetadataKGAgent.yaml similarity index 100% rename from Sandbox/.agentforge/prompts/modules/MetadataKGAgent.yaml rename to sandbox/.agentforge/prompts/modules/MetadataKGAgent.yaml diff --git a/Sandbox/.agentforge/prompts/modules/ToolPrimingAgent.yaml b/sandbox/.agentforge/prompts/modules/ToolPrimingAgent.yaml similarity index 98% rename from Sandbox/.agentforge/prompts/modules/ToolPrimingAgent.yaml rename to sandbox/.agentforge/prompts/modules/ToolPrimingAgent.yaml index 6d31d7be..7b6ee3d6 100644 --- a/Sandbox/.agentforge/prompts/modules/ToolPrimingAgent.yaml +++ b/sandbox/.agentforge/prompts/modules/ToolPrimingAgent.yaml @@ -1,5 +1,5 @@ -Prompts: - System: +prompts: + system: Task: |- You are a tool priming agent tasked with preparing a tool for an objective: @@ -11,7 +11,7 @@ Prompts: To achieve this objective, the following action has been selected: {action} - User: + user: Tool: |+ Your task is to prime the '{tool_name}' tool in the context of the selected action. Instructions explaining how to use the tool are as follows: {tool_info} @@ -45,4 +45,4 @@ Prompts: reasoning: speak: next_tool_context: - ``` \ No newline at end of file + ``` diff --git a/sandbox/.agentforge/settings/models.yaml b/sandbox/.agentforge/settings/models.yaml new file mode 100644 index 00000000..a42e43eb --- /dev/null +++ b/sandbox/.agentforge/settings/models.yaml @@ -0,0 +1,123 @@ +# Default model settings for all agents unless overridden +default_model: + api: gemini_api + model: gemini_flash +# Uncomment to use alternative default models +# api: lm_studio_api +# model: LMStudio +# api: openai_api +# model: omni_model +# model: o1_preview + +# Library of models and parameter defaults +model_library: + openai_api: # API script name + O1Series: # Class name for the API (Case Sensitive) + models: # List of model configurations + o1: + identifier: o1 + params: {} # No overrides for this model + o1_preview: + identifier: o1-preview + o1_mini: + identifier: o1-mini + + params: {} # Default parameters for the model class + + GPT: + models: + omni_model: + identifier: gpt-4o + params: + max_new_tokens: 15000 # Example of overriding parameters + smart_model: + identifier: gpt-4 + smart_fast_model: + identifier: gpt-4-turbo + fast_model: + identifier: gpt-3.5-turbo + + params: # Default parameters for GPT models + max_tokens: 10000 + n: 1 + presence_penalty: 0 + stop: null + temperature: 0.8 + top_p: 0.1 + + anthropic_api: + Claude: + models: + claude3: + identifier: claude-3-opus-20240229 + + params: # Default parameters for Claude models + max_tokens: 10000 + temperature: 0.8 + top_p: 0.1 + + gemini_api: + Gemini: + models: + gemini_pro: + identifier: gemini-1.5-pro + gemini_flash: + identifier: gemini-1.5-flash + + params: # Default parameters for Gemini models + candidate_count: 1 + max_output_tokens: 10000 + temperature: 0.8 + top_k: 40 + top_p: 0.1 + + lm_studio_api: + LMStudio: + models: + llama3_8b: + identifier: lmstudio-community/Meta-Llama-3-8B-Instruct-GGUF + + params: # Default parameters for LMStudio models + host_url: http://localhost:1234/v1/chat/completions + max_tokens: 10000 + stream: false + temperature: 0.8 + + ollama_api: + Ollama: + models: + llama3.1_70b: + identifier: llama3.1:70b + + params: # Default parameters for Ollama models + host_url: http://localhost:11434/api/generate + max_tokens: 10000 + stream: false + temperature: 0.8 + + openrouter_api: + OpenRouter: + models: + phi3med: + identifier: microsoft/phi-3-medium-128k-instruct:free + hermes: + identifier: nousresearch/hermes-3-llama-3.1-405b + reflection: + identifier: mattshumer/reflection-70b:free + + groq_api: + GroqAPI: + models: + llama31: + identifier: llama-3.1-70b-versatile + + params: # Default parameters for GroqAPI models + max_tokens: 10000 + seed: -1 + stop: null + temperature: 0.8 + top_p: 0.1 + +# Embedding library +embedding_library: + library: sentence_transformers diff --git a/sandbox/.agentforge/settings/storage.yaml b/sandbox/.agentforge/settings/storage.yaml new file mode 100644 index 00000000..4fad10bb --- /dev/null +++ b/sandbox/.agentforge/settings/storage.yaml @@ -0,0 +1,39 @@ +# Default storage settings for all agents unless overridden +selected_storage: + implementation: chromadb + configuration: chroma_cosine + +embedding: + selected: distilroberta + +options: + enabled: true + save_memory: true # Note: Saving memory won't work if storage is disabled + iso_timestamp: true # Use ISO format for timestamps + unix_timestamp: true # Use Unix format for timestamps + persist_directory: ./db/ # Relative path for persistent storage + fresh_start: false # Wipes storage on system initialization if true + +library: + chromadb: # Storage implementation + configurations: + chroma_default: {} # No specific settings for this configuration + + chroma_cosine: + metadata: + hnsw_space: cosine + + chroma_ip: + metadata: + hnsw_space: ip + overrides: + persist_directory: ./db/ChromaDB/ip + + # Default settings for the storage implementation + defaults: + persist_directory: ./db/ChromaDB + selected_embedding: distilroberta + +# Embedding library (mapping of embeddings to their identifiers) +embedding_library: + distilroberta: all-distilroberta-v1 diff --git a/sandbox/.agentforge/settings/system.yaml b/sandbox/.agentforge/settings/system.yaml new file mode 100644 index 00000000..979723e2 --- /dev/null +++ b/sandbox/.agentforge/settings/system.yaml @@ -0,0 +1,28 @@ +# Persona settings +persona: + enabled: true + name: default + +# Debug settings +debug: + mode: false + save_memory: false # Save memory during debugging (overrides normal behavior) + simulated_response: "Text designed to simulate an LLM response for debugging purposes + without invoking the model." + +# Logging settings +logging: + enabled: true + console_level: warning + folder: ./logs + files: # Log levels: critical, error, warning, info, debug + agentforge: error + model_io: error + chroma_utils: warning + Actions: warning +misc: + on_the_fly: true # Enables real-time dynamic adjustments for prompts and agent setting overrides + +# System file paths (Read/Write access) +paths: + files: ./files diff --git a/sandbox/.agentforge/tools/BraveSearch.yaml b/sandbox/.agentforge/tools/BraveSearch.yaml new file mode 100644 index 00000000..65dcca26 --- /dev/null +++ b/sandbox/.agentforge/tools/BraveSearch.yaml @@ -0,0 +1,35 @@ +Name: Brave Search +Args: + - query (str) + - count (int, optional) +Command: search +Description: |- + The 'Brave Search' tool performs a web search using the Brave Search API. It retrieves search results based on the provided query. Each result includes the title, URL, description, and any extra snippets. + +Instruction: |- + To use the 'Brave Search' tool, follow these steps: + 1. Call the `search` method with the following arguments: + - `query`: A string representing the search query. + - `count`: (Optional) An integer specifying the number of search results to retrieve. Defaults to 10 if not specified. + 2. The method returns a dictionary containing search results in the keys: + - `'web_results'`: A list of web search results. + - `'video_results'`: A list of video search results (if any). + 3. Each item in `'web_results'` includes: + - `title`: The title of the result. + - `url`: The URL of the result. + - `description`: A brief description of the result. + - `extra_snippets`: (Optional) Additional snippets of information. + 4. Utilize the returned results as needed in your application. + +Example: |- + # Example usage of the Brave Search tool: + brave_search = BraveSearch() + results = brave_search.search(query='OpenAI GPT-4', count=5) + for result in results['web_results']: + print(f"Title: {result['title']}") + print(f"URL: {result['url']}") + print(f"Description: {result['description']}") + print('---') + +Script: .agentforge.tools.brave_search +Class: BraveSearch diff --git a/src/agentforge/setup_files/tools/FileWriter.yaml b/sandbox/.agentforge/tools/FileWriter.yaml old mode 100755 new mode 100644 similarity index 92% rename from src/agentforge/setup_files/tools/FileWriter.yaml rename to sandbox/.agentforge/tools/FileWriter.yaml index 519b0f2c..651951ef --- a/src/agentforge/setup_files/tools/FileWriter.yaml +++ b/sandbox/.agentforge/tools/FileWriter.yaml @@ -1,13 +1,13 @@ -Name: File Writer -Args: - - folder (str) - - file (str) - - text (str) - - mode (str='a') -Command: write_file -Description: >- - The 'File Writer' tool writes the provided text to a specified file. You can specify the folder, filename, and the mode (append or overwrite). -Example: response = write_file(folder, file, text, mode='a') -Instruction: >- - The 'write_file' method requires a folder, file name, and the text you want to write as inputs. An optional mode parameter can be provided to decide whether to append ('a') or overwrite ('w') the file. By default, the function appends to the file. -Script: .agentforge.tools.WriteFile +Name: File Writer +Args: + - folder (str) + - file (str) + - text (str) + - mode (str='a') +Command: write_file +Description: >- + The 'File Writer' tool writes the provided text to a specified file. You can specify the folder, filename, and the mode (append or overwrite). +Example: response = write_file(folder, file, text, mode='a') +Instruction: >- + The 'write_file' method requires a folder, file name, and the text you want to write as inputs. An optional mode parameter can be provided to decide whether to append ('a') or overwrite ('w') the file. By default, the function appends to the file. +Script: .agentforge.tools.file_writer.WriteFile diff --git a/Sandbox/.agentforge/tools/ReadDirectory.yaml b/sandbox/.agentforge/tools/ReadDirectory.yaml old mode 100755 new mode 100644 similarity index 96% rename from Sandbox/.agentforge/tools/ReadDirectory.yaml rename to sandbox/.agentforge/tools/ReadDirectory.yaml index 0b6c6bea..4b961309 --- a/Sandbox/.agentforge/tools/ReadDirectory.yaml +++ b/sandbox/.agentforge/tools/ReadDirectory.yaml @@ -1,16 +1,16 @@ -Name: Read Directory -Args: - - directory_paths (str or list of str) - - max_depth (int, optional) -Command: read_directory -Description: >- - The 'Read Directory' tool prints the structure of a directory or multiple directories in a tree-like format. It visually represents folders and files, and you can specify the depth of the structure to be printed. The tool can handle both a single directory path or a list of directory paths. If a specified path does not exist, the tool will create it. Additionally, it indicates if a directory is empty or if there are more files beyond the specified depth. -Example: >- - # For a single directory - directory_structure = read_directory('/path/to/directory', max_depth=3) - - # For multiple directories - directory_structure = read_directory(['/path/to/directory1', '/path/to/directory2'], max_depth=2) -Instruction: >- - The 'read_directory' method requires either a single directory path (string) or a list of directory paths (list of strings). An optional max_depth parameter can be provided to limit the depth of the directory structure displayed. The method returns a string representing the directory structure. It handles directory creation if the path does not exist and checks if directories are empty. The method includes error handling for permissions and file not found errors. -Script: .agentforge.tools.Directory +Name: Read Directory +Args: + - directory_paths (str or list of str) + - max_depth (int, optional) +Command: read_directory +Description: >- + The 'Read Directory' tool prints the structure of a directory or multiple directories in a tree-like format. It visually represents folders and files, and you can specify the depth of the structure to be printed. The tool can handle both a single directory path or a list of directory paths. If a specified path does not exist, the tool will create it. Additionally, it indicates if a directory is empty or if there are more files beyond the specified depth. +Example: >- + # For a single directory + directory_structure = read_directory('/path/to/directory', max_depth=3) + + # For multiple directories + directory_structure = read_directory(['/path/to/directory1', '/path/to/directory2'], max_depth=2) +Instruction: >- + The 'read_directory' method requires either a single directory path (string) or a list of directory paths (list of strings). An optional max_depth parameter can be provided to limit the depth of the directory structure displayed. The method returns a string representing the directory structure. It handles directory creation if the path does not exist and checks if directories are empty. The method includes error handling for permissions and file not found errors. +Script: .agentforge.tools.directory.Directory diff --git a/sandbox/.agentforge/tools/SemanticChunk.yaml b/sandbox/.agentforge/tools/SemanticChunk.yaml new file mode 100644 index 00000000..62d0e603 --- /dev/null +++ b/sandbox/.agentforge/tools/SemanticChunk.yaml @@ -0,0 +1,32 @@ +Name: Semantic Chunk +Args: + - text (str) + - min_length (int, optional) + - max_length (int, optional) +Command: semantic_chunk +Description: |- + The 'Semantic Chunk' tool splits input text into semantically meaningful chunks based on content. +Instruction: |- + To use the 'Semantic Chunk' tool, follow these steps: + 1. Call the `semantic_chunk` function with the following arguments: + - `text`: The text you want to split into chunks. + - `min_length`: (Optional) The minimum length of each chunk in characters. Defaults to 200. + - `max_length`: (Optional) The maximum length of each chunk in characters. Defaults to 2000. + 2. The function returns a list of text chunks. + 3. Use the chunks as needed in your application. +Example: |- + # Example usage of the Semantic Chunk tool: + from agentforge.tools.SemanticChunk import semantic_chunk + + text = "Your long text goes here..." + + # Split text into chunks with default lengths + chunks = semantic_chunk(text) + print(f"Number of chunks: {len(chunks)}") + for i, chunk in enumerate(chunks, 1): + print(f"Chunk {i}: {chunk[:50]}...") + + # Split text with custom chunk lengths + chunks = semantic_chunk(text, min_length=100, max_length=1000) + print(f"Number of chunks with custom sizes: {len(chunks)}") +Script: .agentforge.tools.semantic_chunk.SemanticChunk diff --git a/src/agentforge/setup_files/tools/WebScrape.yaml b/sandbox/.agentforge/tools/WebScrape.yaml old mode 100755 new mode 100644 similarity index 91% rename from src/agentforge/setup_files/tools/WebScrape.yaml rename to sandbox/.agentforge/tools/WebScrape.yaml index c25d3861..50711091 --- a/src/agentforge/setup_files/tools/WebScrape.yaml +++ b/sandbox/.agentforge/tools/WebScrape.yaml @@ -1,9 +1,10 @@ -Name: Web Scrape -Args: url (str) -Command: get_plain_text -Description: >- - The 'Web Scrape' tool is used to pull all text from a webpage. Simply provide the web address (URL), and the tool will return the webpage's content in plain text. -Example: scrapped = get_plain_text(url) -Instruction: >- - The 'get_plain_text' method of the 'Web Scrape' tool takes a URL as an input, which represents the webpage to scrape. It returns the textual content of that webpage as a string. You can send only one URL, so if you receive more than one, choose the most likely URL to contain the results you expect. -Script: .agentforge.tools.WebScrape +Name: Web Scrape +Args: url (str) +Command: get_plain_text +Description: >- + The 'Web Scrape' tool is used to pull all text from a webpage. Simply provide the web address (URL), and the tool will return the webpage's content in plain text. +Example: scrapped = get_plain_text(url) +Instruction: >- + The 'get_plain_text' method of the 'Web Scrape' tool takes a URL as an input, which represents the webpage to scrape. It returns the textual content of that webpage as a string. You can send only one URL, so if you receive more than one, choose the most likely URL to contain the results you expect. +Script: .agentforge.tools.web_scrape +Class: WebScrape diff --git a/Sandbox/CustomAgents/DocsAgent.py b/sandbox/CustomAgents/CustAgent.py similarity index 51% rename from Sandbox/CustomAgents/DocsAgent.py rename to sandbox/CustomAgents/CustAgent.py index 620b2cb4..4f676331 100644 --- a/Sandbox/CustomAgents/DocsAgent.py +++ b/sandbox/CustomAgents/CustAgent.py @@ -1,5 +1,4 @@ from agentforge.agent import Agent - -class DocsAgent(Agent): - pass +class CustAgent(Agent): + pass \ No newline at end of file diff --git a/Sandbox/KGTest.py b/sandbox/KGTest.py similarity index 81% rename from Sandbox/KGTest.py rename to sandbox/KGTest.py index cfbc92ea..b9592898 100644 --- a/Sandbox/KGTest.py +++ b/sandbox/KGTest.py @@ -1,16 +1,16 @@ from agentforge.modules.KnowledgeTraversal import KnowledgeTraversal -from agentforge.utils.Logger import Logger +from agentforge.utils.logger import Logger logger = Logger(name="KGTest") if __name__ == "__main__": kg = KnowledgeTraversal() kg_name = 'knowledge_graph' - query = "llm" + query = "apis" search_map = {"predicate": "predicate"} print(f"Metadata Map:{search_map}") - # result = kg.query_knowledge(kg_name, "llm", search_map, 3, 3) + # result = kg.query_knowledge(kg_name, "apis", search_map, 3, 3) result = kg.query_knowledge(knowledge_base_name=kg_name, query=query, metadata_map=search_map, diff --git a/src/agentforge/modules/LearnDoc.py b/sandbox/LearnDoc.py old mode 100755 new mode 100644 similarity index 94% rename from src/agentforge/modules/LearnDoc.py rename to sandbox/LearnDoc.py index 48264cd4..cd7a9c31 --- a/src/agentforge/modules/LearnDoc.py +++ b/sandbox/LearnDoc.py @@ -1,100 +1,102 @@ -from agentforge.agents.LearnKGAgent import LearnKGAgent -from agentforge.tools.GetText import GetText -from agentforge.tools.IntelligentChunk import intelligent_chunk -from agentforge.modules.InjectKG import Consume -from agentforge.utils.Logger import Logger -from agentforge.tools.CleanString import Strip -from ..utils.ChromaUtils import ChromaUtils -import os - - -class FileProcessor: - """ - Processes files to extract text, chunk it intelligently, learn from it using a knowledge graph agent, - and inject the learned knowledge into a database. - - The class orchestrates a pipeline involving reading text from files, dividing the text into manageable chunks, - processing those chunks to extract knowledge, and finally, storing this knowledge. It leverages a series of - tools and agents, including GetText for reading files, intelligent_chunk for chunking text, LearnKGAgent for - processing text chunks, and InjectKG.Consume for database injection. - """ - def __init__(self): - """ - Initializes the FileProcessor class with its required components. - """ - self.logger = Logger(name=self.__class__.__name__) - - self.intelligent_chunk = intelligent_chunk - self.get_text = GetText() - self.learn_kg = LearnKGAgent() - self.consumer = Consume() - self.strip = Strip() - self.store = ChromaUtils() - - def process_file(self, knowledge_base_name: str, file_path: str) -> None: - """ - Processes a single file through the pipeline, extracting text, chunking it, learning from the chunks, - and injecting the learned knowledge into the knowledge base. - - Parameters: - knowledge_base_name (str): The name of the knowledge base where the extracted information will be stored. - file_path (str): The file path of the file to process. - - Returns: - None - - This method sequentially performs the following steps: - 1. Reads text from the provided file. - 2. Chunks the text into smaller, more manageable pieces. - 3. Processes each chunk with the LearnKGAgent to learn and extract sentences. - 4. Each sentence is then processed for knowledge extraction. - 5. Extracted knowledge is injected into the knowledge base using the 'consume' method of the consumer tool. - - Errors at any stage are logged, and processing can optionally continue to the next chunk if an error occurs. - The filename without its extension is used as the source name in the knowledge base entries. - - The method does not return a value but may print information about the injected knowledge entries - and log errors. - """ - filename_with_extension = os.path.basename(file_path) - filename_without_extension, _ = os.path.splitext(filename_with_extension) - - try: - # Step 1: Extract text from the file - file_content = self.get_text.read_file(file_path) - file_clean = self.strip.strip_invalid_chars(file_content) - except Exception as e: - self.logger.log(f"Error reading file: {e}", 'error') - return - - try: - # Step 2: Create chunks of the text - chunks = self.intelligent_chunk(file_clean, chunk_size=2) - except Exception as e: - self.logger.log(f"Error chunking text: {e}", 'error') - return - - for chunk in chunks: - try: - # Steps within the loop for learning and injecting data - kg_results = self.store.query_memory(chunk) - data = self.learn_kg.run(text_chunk=chunk, existing_knowledge=kg_results) - - if data is not None and 'sentences' in data and data['sentences']: - for key in data['sentences']: - sentence = data['sentences'][key] - reason = data['reasons'].get(key, "") - - injected = self.consumer.consume(knowledge_base_name=knowledge_base_name, - sentence=sentence, - reason=reason, - source_name=filename_without_extension, - source_path=file_path, - chunk=chunk, - existing_knowledge=kg_results) - print(f"The following entry was added to the knowledge graph:\n{injected}\n\n") - else: - self.logger.log("No relevant knowledge was found", 'info') - except Exception as e: - self.logger.log(f"Error processing chunk: {e}", 'error') - continue # Optionally continue to the next chunk + + +from agentforge.agents.LearnKGAgent import LearnKGAgent +from agentforge.tools.get_text import GetText +from agentforge.tools.intelligent_chunk import intelligent_chunk +from agentforge.modules.InjectKG import Consume +from agentforge.utils.logger import Logger +from agentforge.tools.clean_string import Strip +from ..utils.ChromaUtils import ChromaUtils +import os + + +class FileProcessor: + """ + Processes files to extract text, chunk it intelligently, learn from it using a knowledge graph agent, + and inject the learned knowledge into a database. + + The class orchestrates a pipeline involving reading text from files, dividing the text into manageable chunks, + processing those chunks to extract knowledge, and finally, storing this knowledge. It leverages a series of + tools and agents, including GetText for reading files, intelligent_chunk for chunking text, LearnKGAgent for + processing text chunks, and InjectKG.Consume for database injection. + """ + def __init__(self): + """ + Initializes the FileProcessor class with its required components. + """ + self.logger = Logger(name=self.__class__.__name__) + + self.intelligent_chunk = intelligent_chunk + self.get_text = GetText() + self.learn_kg = LearnKGAgent() + self.consumer = Consume() + self.strip = Strip() + self.store = ChromaUtils() + + def process_file(self, knowledge_base_name: str, file_path: str) -> None: + """ + Processes a single file through the pipeline, extracting text, chunking it, learning from the chunks, + and injecting the learned knowledge into the knowledge base. + + Parameters: + knowledge_base_name (str): The name of the knowledge base where the extracted information will be stored. + file_path (str): The file path of the file to process. + + Returns: + None + + This method sequentially performs the following steps: + 1. Reads text from the provided file. + 2. Chunks the text into smaller, more manageable pieces. + 3. Processes each chunk with the LearnKGAgent to learn and extract sentences. + 4. Each sentence is then processed for knowledge extraction. + 5. Extracted knowledge is injected into the knowledge base using the 'consume' method of the consumer tool. + + Errors at any stage are logged, and processing can optionally continue to the next chunk if an error occurs. + The filename without its extension is used as the source name in the knowledge base entries. + + The method does not return a value but may print information about the injected knowledge entries + and log errors. + """ + filename_with_extension = os.path.basename(file_path) + filename_without_extension, _ = os.path.splitext(filename_with_extension) + + try: + # Step 1: Extract text from the file + file_content = self.get_text.read_file(file_path) + file_clean = self.strip.strip_invalid_chars(file_content) + except Exception as e: + self.logger.log(f"Error reading file: {e}", 'error') + return + + try: + # Step 2: Create chunks of the text + chunks = self.intelligent_chunk(file_clean, chunk_size=2) + except Exception as e: + self.logger.log(f"Error chunking text: {e}", 'error') + return + + for chunk in chunks: + try: + # Steps within the loop for learning and injecting data + kg_results = self.store.query_memory(chunk) + data = self.learn_kg.run(text_chunk=chunk, existing_knowledge=kg_results) + + if data is not None and 'sentences' in data and data['sentences']: + for key in data['sentences']: + sentence = data['sentences'][key] + reason = data['reasons'].get(key, "") + + injected = self.consumer.consume(knowledge_base_name=knowledge_base_name, + sentence=sentence, + reason=reason, + source_name=filename_without_extension, + source_path=file_path, + chunk=chunk, + existing_knowledge=kg_results) + print(f"The following entry was added to the knowledge graph:\n{injected}\n\n") + else: + self.logger.log("No relevant knowledge was found", 'info') + except Exception as e: + self.logger.log(f"Error processing chunk: {e}", 'error') + continue # Optionally continue to the next chunk diff --git a/sandbox/RunTestAgent.py b/sandbox/RunTestAgent.py new file mode 100644 index 00000000..f841216d --- /dev/null +++ b/sandbox/RunTestAgent.py @@ -0,0 +1,8 @@ +from agentforge.agent import Agent + +test = Agent(agent_name="TestAgent") +text = "Hi! I am testing your functionality, is everything nominal and in order from your point of view?" + +result = test.run(text=text) +print(f"TestAgent Response: {result}") + diff --git a/sandbox/RunTestFlow.py b/sandbox/RunTestFlow.py new file mode 100644 index 00000000..1e373b46 --- /dev/null +++ b/sandbox/RunTestFlow.py @@ -0,0 +1,8 @@ +from agentforge.cogarch import CogArch + +test = CogArch(name="simple") +text = "Hi! I am testing your functionality, is everything nominal and in order from your point of view?" + +result = test.run(text=text) +print(f"Test Flow Response: {result}") + diff --git a/Sandbox/TestActions.py b/sandbox/TestActions.py similarity index 83% rename from Sandbox/TestActions.py rename to sandbox/TestActions.py index 50236c41..7ce5f7a6 100644 --- a/Sandbox/TestActions.py +++ b/sandbox/TestActions.py @@ -1,4 +1,4 @@ -from agentforge.modules.Actions import Actions +from agentforge.modules.actions import Actions objective = 'Find news about the Ukraine war' context = 'make sure that the news is within the last 15 days. Provide links to sources' diff --git a/Sandbox/TestChroma.py b/sandbox/TestChroma.py similarity index 56% rename from Sandbox/TestChroma.py rename to sandbox/TestChroma.py index ac86b65f..8c179a26 100644 --- a/Sandbox/TestChroma.py +++ b/sandbox/TestChroma.py @@ -1,6 +1,6 @@ -from agentforge.utils.ChromaUtils import ChromaUtils +from agentforge.storage.chroma_storage import ChromaStorage -storage = ChromaUtils() +storage = ChromaStorage() print('hi') var = storage.search_storage_by_threshold("Actions","search web", threshold=0.8, num_results=5) diff --git a/Sandbox/TestLLM.py b/sandbox/TestLLM.py similarity index 84% rename from Sandbox/TestLLM.py rename to sandbox/TestLLM.py index f0e9a16d..3bac8040 100644 --- a/Sandbox/TestLLM.py +++ b/sandbox/TestLLM.py @@ -1,6 +1,4 @@ from CustomAgents.DocsAgent import DocsAgent -from agentforge.utils.ChromaUtils import ChromaUtils -from agentforge.utils.function_utils import Logger docs_agent = DocsAgent() # kb = ChromaUtils() diff --git a/Sandbox/TestLearn.py b/sandbox/TestLearn.py similarity index 100% rename from Sandbox/TestLearn.py rename to sandbox/TestLearn.py diff --git a/Sandbox/chat_with_docs.py b/sandbox/chat_with_docs.py similarity index 64% rename from Sandbox/chat_with_docs.py rename to sandbox/chat_with_docs.py index 273dfa88..61148650 100644 --- a/Sandbox/chat_with_docs.py +++ b/sandbox/chat_with_docs.py @@ -1,9 +1,8 @@ -from CustomAgents.DocsAgent import DocsAgent -from agentforge.utils.ChromaUtils import ChromaUtils -from agentforge.utils.function_utils import Logger +from agentforge.agent import Agent +from agentforge.storage.chroma_storage import ChromaStorage -docs_agent = DocsAgent() -# kb = ChromaUtils() +docs_agent = Agent(agent_name="DocsAgent") +kb = ChromaStorage() while True: user_input = input("Welcome to the chat with docs!\nQuestion: ") diff --git a/Sandbox/chatbot.py b/sandbox/chatbot.py similarity index 100% rename from Sandbox/chatbot.py rename to sandbox/chatbot.py diff --git a/Sandbox/checkenv.py b/sandbox/checkenv.py similarity index 100% rename from Sandbox/checkenv.py rename to sandbox/checkenv.py diff --git a/Sandbox/claud3test.py b/sandbox/claud3test.py similarity index 100% rename from Sandbox/claud3test.py rename to sandbox/claud3test.py diff --git a/Sandbox/docs/2406.04271v1.pdf b/sandbox/docs/2406.04271v1.pdf similarity index 100% rename from Sandbox/docs/2406.04271v1.pdf rename to sandbox/docs/2406.04271v1.pdf diff --git a/sandbox/modules/TestActions.py b/sandbox/modules/TestActions.py new file mode 100644 index 00000000..99075395 --- /dev/null +++ b/sandbox/modules/TestActions.py @@ -0,0 +1,3 @@ +from agentforge.modules.actions import Action + +test = Action() diff --git a/Sandbox/CustomAgents/__init__.py b/sandbox/modules/__init__.py similarity index 100% rename from Sandbox/CustomAgents/__init__.py rename to sandbox/modules/__init__.py diff --git a/Sandbox/modules/discord_client.py b/sandbox/modules/discord_client.py similarity index 98% rename from Sandbox/modules/discord_client.py rename to sandbox/modules/discord_client.py index 324a198b..8146f73c 100644 --- a/Sandbox/modules/discord_client.py +++ b/sandbox/modules/discord_client.py @@ -1,10 +1,10 @@ -# modules/DiscordClient.py +# modules/discord_client.py import discord import os import asyncio import threading -from agentforge.utils.Logger import Logger +from agentforge.utils.logger import Logger class DiscordClient: diff --git a/Sandbox/modules/read_docs.py b/sandbox/modules/read_docs.py similarity index 80% rename from Sandbox/modules/read_docs.py rename to sandbox/modules/read_docs.py index 196399b4..dba8c8e4 100644 --- a/Sandbox/modules/read_docs.py +++ b/sandbox/modules/read_docs.py @@ -1,11 +1,11 @@ -from agentforge.tools.SemanticChunk import semantic_chunk -from agentforge.tools.GetText import GetText -from agentforge.utils.ChromaUtils import ChromaUtils +from agentforge.tools.semantic_chunk import semantic_chunk +from agentforge.tools.get_text import GetText +from agentforge.storage.chroma_storage import ChromaStorage import os gettext_instance = GetText() folder = '../docs' -storage = ChromaUtils('dignity') +storage = ChromaStorage('dignity') def list_files(directory): diff --git a/setup.py b/setup.py index 659ccc9e..1e222e85 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ from setuptools import setup, find_packages +__version__ = "0.5.0" LICENSE = "GNU General Public License v3 or later (GPLv3+)" @@ -10,7 +11,7 @@ def get_long_description(): setup( name="agentforge", - version="0.4.0", + version="0.5.0", description="AI-driven task automation system", author="John Smith, Ansel Anselmi", author_email="contact@agentforge.net", @@ -18,9 +19,15 @@ def get_long_description(): include_package_data=True, packages=find_packages(where="src"), package_dir={"": "src"}, + entry_points={ + "console_scripts": [ + "agentforge=agentforge.cli:main" + ] + }, install_requires=[ - "chromadb==0.5.3", - # "numpy==1.26.4", + "chromadb==0.6.2", + "numpy<2.0.0; python_version<'3.12'", + "numpy>=2.0.0; python_version>='3.12'", "sentence-transformers", "wheel", "groq", @@ -30,13 +37,18 @@ def get_long_description(): "termcolor==2.4.0", "openai", "anthropic", - # "google-api-python-client", + "google-api-python-client", "beautifulsoup4", "browse", "scipy", "discord.py", "semantic-text-splitter", "google-generativeai", + "PyYAML", + "ruamel.yaml", + "requests", + "ruamel.yaml", + "xmltodict", "setuptools>=70.0.0 ", # not directly required, pinned by Snyk to avoid a vulnerability ], extras_require={ @@ -57,20 +69,13 @@ def get_long_description(): "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ], python_requires=">=3.9", package_data={ - 'agentforge.utils.guiutils': ['DiscordClient.py'], + 'agentforge.utils.guiutils': ['discord_client.py'], '': ['*.yaml'], # Include your file types as needed }, - # package_data={ - # ".agentforge.utils.setup_files": ["*", "**/*"], - # ".agentforge.utils.guiutils": ["*", "**/*"], - # }, - # entry_points={ - # 'console_scripts': [ - # '.agentforge=.agentforge.utils.setup_files.agentforge_cli:main', - # ], - # } ) diff --git a/src/agentforge/agent.py b/src/agentforge/agent.py index c9f603f3..d89c62b2 100755 --- a/src/agentforge/agent.py +++ b/src/agentforge/agent.py @@ -1,29 +1,47 @@ -from typing import Any, Dict, List, Optional -from agentforge.llm import LLM -from agentforge.utils.Logger import Logger from .config import Config -from agentforge.utils.PromptHandling import PromptHandling - +from agentforge.apis.base_api import BaseModel +from agentforge.utils.logger import Logger +from agentforge.utils.prompt_processor import PromptProcessor +from typing import Any, Dict, Optional class Agent: - def __init__(self): + def __init__(self, agent_name: Optional[str] = None, log_file: Optional[str] = 'agentforge'): """ Initializes an Agent instance, setting up its name, logger, data attributes, and agent-specific configurations. It attempts to load the agent's configuration data and storage settings. + + Args: + agent_name (Optional[str]): The name of the agent. If not provided, the class name is used. + log_file (Optional[str]): The name of the log file for the agent. If not provided, AgentForge.log is used. """ - self.agent_name: str = self.__class__.__name__ - self.logger: Logger = Logger(name=self.agent_name) + # Set agent_name to the provided name or default to the class name + self.agent_name: str = agent_name if agent_name is not None else self.__class__.__name__ + + # Initialize logger with the agent's name + self.logger: Logger = Logger(self.agent_name, log_file) + + # Initialize configurations and handlers self.config = Config() - self.prompt_handling = PromptHandling() + self.prompt_processor = PromptProcessor() - self.data: Dict[str, Any] = {} - self.prompt: Optional[List[str]] = None + # Initialize data attributes + self.agent_data: Optional[Dict[str, Any]] = None + self.persona: Optional[Dict[str, Any]] = None + self.model: Optional[BaseModel] = None + self.prompt_template: Optional[Dict[str, Any]] = None + self.template_data: Dict[str, Any] = {} + self.prompt: Optional[Dict[str]] = None self.result: Optional[str] = None self.output: Optional[str] = None + self.images: Optional[list[str]] = [] + + # Load and validate agent data during initialization + self.initialize_agent_config() - if not hasattr(self, 'agent_data'): # Prevent re-initialization - self.agent_data: Optional[Dict[str, Any]] = None + # --------------------------------- + # Execution + # --------------------------------- def run(self, **kwargs: Any) -> Optional[str]: """ @@ -37,74 +55,142 @@ def run(self, **kwargs: Any) -> Optional[str]: Optional[str]: The output generated by the agent or None if an error occurred during execution. """ try: - self.logger.log(f"\n{self.agent_name} - Running...", 'info') + self.logger.info(f"{self.agent_name} - Running...") self.load_data(**kwargs) self.process_data() - self.generate_prompt() + self.render_prompt() self.run_llm() self.parse_result() self.save_to_storage() self.build_output() - self.data = {} - self.logger.log(f"\n{self.agent_name} - Done!", 'info') + self.logger.info(f"{self.agent_name} - Done!") except Exception as e: - self.logger.log(f"Agent execution failed: {e}", 'error') + self.logger.error(f"Agent execution failed: {e}") return None - return self.output - def load_data(self, **kwargs: Any) -> None: - """ - Central method for data loading that orchestrates the loading of agent data, persona-specific data, - storage data, and any additional data. + # --------------------------------- + # Configuration Loading + # --------------------------------- - Parameters: - **kwargs (Any): Keyword arguments for additional data loading. + def initialize_agent_config(self) -> None: + """ + Loads the agent's configuration data and resolves it's storage. """ self.load_agent_data() + self.load_prompt_template() self.load_persona_data() + self.load_model() self.resolve_storage() - self.load_from_storage() - self.load_additional_data() - self.load_kwargs(**kwargs) - def load_kwargs(self, **kwargs: Any) -> None: + def load_agent_data(self) -> None: """ - Loads the variables passed to the agent as data. - - Parameters: - **kwargs (Any): Additional keyword arguments to be merged into the agent's data. + Loads the agent's configuration data. """ - try: - for key in kwargs: - self.data[key] = kwargs[key] - except Exception as e: - self.logger.log(f"Error loading kwargs: {e}", 'error') + self.agent_data = self.config.load_agent_data(self.agent_name).copy() + self.validate_agent_data() - def load_agent_data(self) -> None: + def load_prompt_template(self) -> None: """ - Loads the agent's configuration data including parameters and prompts. + Validates that prompts are properly formatted and available. """ - try: - self.agent_data = self.config.load_agent_data(self.agent_name).copy() - self.data.update({ - 'params': self.agent_data.get('params').copy(), - 'prompts': self.agent_data['prompts'].copy() - }) - except Exception as e: - self.logger.log(f"Error loading agent data: {e}", 'error') + self.prompt_template = self.agent_data.get('prompts', {}) + if not self.prompt_template: + error_msg = f"No prompts defined for agent '{self.agent_name}'." + self.logger.error(error_msg) + raise ValueError(error_msg) + + self.prompt_processor.check_prompt_format(self.prompt_template) + self.logger.debug(f"Prompts for '{self.agent_name}' validated.") def load_persona_data(self) -> None: """ - Loads the persona data for the agent if available. Will not load persona data if personas is disabled in system settings. + Loads and validates the persona data for the agent if available. Will not load persona data if personas is disabled in system settings. """ - if not self.agent_data['settings']['system'].get('PersonasEnabled'): - return None + personas_enabled = self.agent_data['settings']['system']['persona'].get('enabled', False) + if personas_enabled: + self.persona = self.agent_data.get('persona', {}) + self.validate_persona_data() + self.logger.debug(f"Persona Data Loaded for '{self.agent_name}'.") + + if self.persona: + for key in self.persona: + self.template_data[key.lower()] = self.persona[key] - persona = self.agent_data.get('persona', {}) - if persona: - for key in persona: - self.data[key.lower()] = persona[key] + def load_model(self): + self.model = self.agent_data.get('model') + if not self.model: + error_msg = f"Model not specified for agent '{self.agent_name}'." + self.logger.error(error_msg) + raise ValueError(error_msg) + + def resolve_storage(self): + """ + Initializes the storage for the agent, if storage is enabled. + """ + storage_enabled = self.agent_data['settings']['storage']['options'].get('enabled', False) + if not storage_enabled: + self.agent_data['storage'] = None + return + + # Needs rework + # from .utils.ChromaUtils import ChromaUtils + # persona_name = self.persona.get('Name', 'DefaultPersona') if self.persona else 'DefaultPersona' + # self.agent_data['storage'] = ChromaUtils(persona_name) + + # --------------------------------- + # Validation + # --------------------------------- + + def validate_agent_data(self) -> None: + """ + Validates that agent_data has the necessary structure and keys. + """ + if not self.agent_data: + error_msg = f"Agent data for '{self.agent_name}' is not loaded." + self.logger.error(error_msg) + raise ValueError(error_msg) + + required_keys = ['params', 'prompts', 'settings'] + for key in required_keys: + if key not in self.agent_data: + error_msg = f"Agent data missing required key '{key}' for agent '{self.agent_name}'." + self.logger.error(error_msg) + raise ValueError(error_msg) + + if 'system' not in self.agent_data['settings']: + error_msg = f"Agent data settings missing 'system' key for agent '{self.agent_name}'." + self.logger.error(error_msg) + raise ValueError(error_msg) + + def validate_persona_data(self) -> None: + """ + Validates persona data. + """ + if not self.persona: + self.logger.warning(f"Personas are enabled but no persona data found for agent '{self.agent_name}'.") + + self.logger.debug(f"Persona for '{self.agent_name}' validated!.") + # Note: We may want to implement additional persona data validation later on. + + # --------------------------------- + # Data Loading + # --------------------------------- + + def load_data(self, **kwargs: Any) -> None: + """ + Central method for data loading that orchestrates the loading of agent data, persona-specific data, + storage data, and any additional data. + + Parameters: + **kwargs (Any): Keyword arguments for additional data loading. + """ + if self.agent_data['settings']['system']['misc'].get('on_the_fly', False): + self.initialize_agent_config() + + self.load_from_storage() + self.load_additional_data() + self.template_data.update(kwargs) def load_from_storage(self) -> None: """ @@ -113,62 +199,57 @@ def load_from_storage(self) -> None: Notes: - The storage instance for an Agent is set at self.agent_data['storage']. - - The 'StorageEnabled' setting is the system.yaml file must be set to 'True'. + - The 'StorageEnabled' setting in the system.yaml file must be set to 'True'. """ pass def load_additional_data(self) -> None: """ - Placeholder for loading additional data. Meant to be overridden by custom agents as needed. + Placeholder for loading additional data to the prompt template data. Meant to be overridden by custom agents + as needed. """ pass + # --------------------------------- + # Processing + # --------------------------------- + def process_data(self) -> None: """ Placeholder for data processing. Meant to be overridden by custom agents for specific data processing needs. """ pass - def generate_prompt(self) -> None: + def render_prompt(self) -> None: """ Generates the prompts for the language model based on the template data. """ - try: - prompts = self.data.get('prompts', {}) + self.prompt = self.prompt_processor.render_prompts(self.prompt_template, self.template_data) + self.prompt_processor.validate_rendered_prompts(self.prompt) # {'System': '...', 'User': '...'} - self.prompt_handling.check_prompt_format(prompts) - rendered_prompts = self.prompt_handling.render_prompts(prompts, self.data) - self.prompt_handling.validate_rendered_prompts(rendered_prompts) - self.prompt = rendered_prompts # {'System': '...', 'User': '...'} - except Exception as e: - self.logger.log(f"Error generating prompt: {e}", 'error') - self.prompt = None - raise + # def process_images(self): + + # --------------------------------- + # LLM Execution + # --------------------------------- def run_llm(self) -> None: """ Executes the language model generation with the generated prompt(s) and any specified parameters. """ - try: - model: LLM = self.agent_data['llm'] - params: Dict[str, Any] = self.agent_data.get("params", {}) - params['agent_name'] = self.agent_name - self.result = model.generate_text(self.prompt, **params).strip() - except Exception as e: - self.logger.log(f"Error running LLM: {e}", 'error') - self.result = None - - def resolve_storage(self): - """ - Initializes the storage for the agent, if storage is enabled. + if self.agent_data['settings']['system']['debug'].get('mode', False): + self.result = self.agent_data['simulated_response'] + return - Returns: None - """ - if not self.agent_data['settings']['system'].get('StorageEnabled'): - return None + params: Dict[str, Any] = self.agent_data.get("params", {}) + params['agent_name'] = self.agent_name + if self.images and len(self.images) > 0: + params['images'] = self.images + self.result = self.model.generate(self.prompt, **params).strip() - from .utils.ChromaUtils import ChromaUtils - self.agent_data['storage'] = ChromaUtils(self.agent_data['persona']['Name']) + # --------------------------------- + # Result Handling + # --------------------------------- def parse_result(self) -> None: """ @@ -184,7 +265,7 @@ def save_to_storage(self) -> None: Notes: - The storage instance for an Agent is set at self.agent_data['storage']. - - The 'StorageEnabled' setting is the system.yaml file must be set to 'True'. + - The 'StorageEnabled' setting in the system.yaml file must be set to 'True'. """ pass diff --git a/src/agentforge/agents/ActionCreationAgent.py b/src/agentforge/agents/ActionCreationAgent.py deleted file mode 100755 index ac5e5cd7..00000000 --- a/src/agentforge/agents/ActionCreationAgent.py +++ /dev/null @@ -1,5 +0,0 @@ -from agentforge.agent import Agent - - -class ActionCreationAgent(Agent): - pass diff --git a/src/agentforge/agents/ActionSelectionAgent.py b/src/agentforge/agents/ActionSelectionAgent.py deleted file mode 100755 index a7840d65..00000000 --- a/src/agentforge/agents/ActionSelectionAgent.py +++ /dev/null @@ -1,5 +0,0 @@ -from agentforge.agent import Agent - - -class ActionSelectionAgent(Agent): - pass diff --git a/src/agentforge/agents/LearnKGAgent.py b/src/agentforge/agents/LearnKGAgent.py deleted file mode 100755 index a1daa3a5..00000000 --- a/src/agentforge/agents/LearnKGAgent.py +++ /dev/null @@ -1,20 +0,0 @@ -from agentforge.agent import Agent -from agentforge.utils.ParsingUtils import ParsingUtils - -class LearnKGAgent(Agent): - def build_output(self): - """ - Overrides the build_output method from the Agent class to parse the result string into a structured format. - - This method attempts to parse the result (assumed to be in YAML format) using the agent's utility functions - and sets the parsed output as the agent's output. If parsing fails, it logs the error using the agent's - logger. - - Raises: - Exception: If there's an error during parsing, it logs the error using a specialized logging method - provided by the logger and re-raises the exception to signal failure to the calling context. - """ - try: - self.output = ParsingUtils().parse_yaml_content(self.result) - except Exception as e: - self.logger.parsing_error(self.result, e) diff --git a/src/agentforge/agents/MetadataKGAgent.py b/src/agentforge/agents/MetadataKGAgent.py deleted file mode 100755 index fee338b1..00000000 --- a/src/agentforge/agents/MetadataKGAgent.py +++ /dev/null @@ -1,21 +0,0 @@ -from agentforge.agent import Agent -from agentforge.utils.ParsingUtils import ParsingUtils - - -class MetadataKGAgent(Agent): - def build_output(self): - """ - Overrides the build_output method from the Agent class to parse the result string into a structured format. - - This method attempts to parse the result (assumed to be in YAML format) using the agent's utility functions - and sets the parsed output as the agent's output. If parsing fails, it logs the error using the agent's - logger. - - Raises: - Exception: If there's an error during parsing, it logs the error using a specialized logging method - provided by the logger and re-raises the exception to signal failure to the calling context. - """ - try: - self.output = ParsingUtils().parse_yaml_content(self.result) - except Exception as e: - self.logger.parsing_error(self.result, e) diff --git a/src/agentforge/agents/ToolPrimingAgent.py b/src/agentforge/agents/ToolPrimingAgent.py deleted file mode 100644 index 15390a00..00000000 --- a/src/agentforge/agents/ToolPrimingAgent.py +++ /dev/null @@ -1,5 +0,0 @@ -from agentforge.agent import Agent - - -class ToolPrimingAgent(Agent): - pass diff --git a/Sandbox/modules/__init__.py b/src/agentforge/apis/__init__.py old mode 100644 new mode 100755 similarity index 100% rename from Sandbox/modules/__init__.py rename to src/agentforge/apis/__init__.py diff --git a/src/agentforge/apis/anthropic_api.py b/src/agentforge/apis/anthropic_api.py new file mode 100755 index 00000000..b61ac7a0 --- /dev/null +++ b/src/agentforge/apis/anthropic_api.py @@ -0,0 +1,34 @@ +import os +import anthropic +from .base_api import BaseModel + +API_KEY = os.getenv('ANTHROPIC_API_KEY') +client = anthropic.Anthropic(api_key=API_KEY) + + +class Claude(BaseModel): + """ + A class for interacting with Anthropic's Claude models to generate text based on provided prompts. + + Manages API calls to Anthropic, handling errors such as rate limits, and retries failed requests with exponential + backoff. + """ + + @staticmethod + def _prepare_prompt(model_prompt): + return { + 'messages': [{'role': 'user', 'content': model_prompt.get('user')}], + 'system': model_prompt.get('system') + } + + def _do_api_call(self, prompt, **filtered_params): + response = client.messages.create( + model=self.model_name, + messages=prompt.get('messages'), + system=prompt.get('system'), + **filtered_params + ) + return response + + def _process_response(self, raw_response): + return raw_response.content[0].text \ No newline at end of file diff --git a/src/agentforge/apis/base_api.py b/src/agentforge/apis/base_api.py new file mode 100644 index 00000000..eb6dd10a --- /dev/null +++ b/src/agentforge/apis/base_api.py @@ -0,0 +1,94 @@ +import time +from openai import APIError, RateLimitError, APIConnectionError +from agentforge.utils.logger import Logger + + +class BaseModel: + """ + A base class encapsulating shared logic (e.g., logging, retries, prompt building). + Subclasses must implement the _call_api method, which does the actual work of sending prompts. + """ + + # Defaults you might share across all models + default_retries = 3 + default_backoff = 2 + + def __init__(self, model_name, **kwargs): + self.logger = None + self.allowed_params = None + self.excluded_params = None + self.model_name = model_name + self.num_retries = kwargs.get("num_retries", self.default_retries) + self.base_backoff = kwargs.get("base_backoff", self.default_backoff) + + def generate(self, model_prompt, **params): + """ + Main entry point for generating responses. This method handles retries, + calls _call_api for the actual request, and logs everything. + """ + # Log the prompt once at the beginning + self.logger = Logger(name=params.pop('agent_name', 'NamelessAgent')) + self.logger.log_prompt(model_prompt) + + reply = None + for attempt in range(self.num_retries): + backoff = self.base_backoff ** (attempt + 1) + try: + reply = self._call_api(model_prompt, **params) + # If successful, log and break + self.logger.log_response(reply) + break + except (RateLimitError, APIConnectionError) as e: + self.logger.log(f"Error: {str(e)}. Retrying in {backoff} seconds...", level="warning") + time.sleep(backoff) + except APIError as e: + if getattr(e, "status_code", None) == 502: + self.logger.log(f"Error 502: Bad gateway. Retrying in {backoff} seconds...", level="warning") + time.sleep(backoff) + else: + raise + except Exception as e: + # General catch-all for other providers + self.logger.log(f"Error: {str(e)}. Retrying in {backoff} seconds...", level="warning") + time.sleep(backoff) + + if reply is None: + self.logger.log("Error: All retries exhausted. No response received.", level="critical") + return reply + + @staticmethod + def _prepare_prompt(model_prompt): + # Format system/user messages in the appropriate style + return [ + {"role": "system", "content": model_prompt.get('system')}, + {"role": "user", "content": model_prompt.get('user')} + ] + + def _call_api(self, model_prompt, **params): + # Step 1: Build or adapt the prompt in a way that the target model expects. + prompt = self._prepare_prompt(model_prompt) + + # Step 2: Filter or transform the params so that you only pass what this model actually uses. + filtered_params = self._prepare_params(**params) + + # Step 3: Call the actual API with the refined prompt and parameters. + response = self._do_api_call(prompt, **filtered_params) + return self._process_response(response) + + def _prepare_params(self, **params): + if self.allowed_params: + # Keep only parameters explicitly allowed + return {k: v for k, v in params.items() if k in self.allowed_params} + if self.excluded_params: + # Exclude parameters listed in excluded_params + return {k: v for k, v in params.items() if k not in self.excluded_params} + # If neither allowed nor excluded parameters are defined, pass all params + return params + + def _do_api_call(self, prompt, **filtered_params): + # The actual request to the underlying client + raise NotImplementedError("Subclasses must implement _do_call_api method.") + + def _process_response(self, raw_response): + # Subclasses can process the raw responses as needed + return raw_response diff --git a/src/agentforge/apis/gemini_api.py b/src/agentforge/apis/gemini_api.py new file mode 100755 index 00000000..199bf663 --- /dev/null +++ b/src/agentforge/apis/gemini_api.py @@ -0,0 +1,41 @@ +import os +import time +from .base_api import BaseModel +import google.generativeai as genai +from google.generativeai.types import HarmCategory, HarmBlockThreshold +from agentforge.utils.logger import Logger + +# Get API key from Env +GOOGLE_API_KEY = os.getenv('GOOGLE_API_KEY') +genai.configure(api_key=GOOGLE_API_KEY) + + +class Gemini(BaseModel): + """ + A class for interacting with Google's Generative AI models to generate text based on provided prompts. + + Handles API calls to Google's Generative AI, including error handling for rate limits and retries failed requests. + """ + + @staticmethod + def _prepare_prompt(model_prompt): + return '\n\n'.join([model_prompt.get('system'), model_prompt.get('user')]) + + def _do_api_call(self, prompt, **filtered_params): + model = genai.GenerativeModel(self.model_name) + response = model.generate_content( + prompt, + safety_settings={ + HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE, + HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE, + HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE, + HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE, + }, + generation_config=genai.types.GenerationConfig(**filtered_params) + ) + + return response + + def _process_response(self, raw_response): + return raw_response.text + diff --git a/src/agentforge/apis/gemini_with_vision.py b/src/agentforge/apis/gemini_with_vision.py new file mode 100644 index 00000000..bdf00534 --- /dev/null +++ b/src/agentforge/apis/gemini_with_vision.py @@ -0,0 +1,314 @@ +import os +import time +import google.generativeai as genai +from google.generativeai.types import HarmCategory, HarmBlockThreshold +from agentforge.utils.logger import Logger +import base64 +import numpy as np +from PIL import Image +from typing import Union, List, Any +from io import BytesIO +from .base_api import BaseModel +import requests + +# Get API key from Env +GOOGLE_API_KEY = os.getenv('GOOGLE_API_KEY') +genai.configure(api_key=GOOGLE_API_KEY) + + +class Gemini_With_Vision(BaseModel): + """ + A class for interacting with Google's Generative AI models to generate text based on provided prompts and images. + + This class extends BaseModel to handle both text and vision inputs for Gemini models. It supports various image input + formats and provides comprehensive error handling and validation. + + Args: + model_name (str): The name of the Gemini model to use (e.g., "gemini-1.5-flash", "gemini-pro-vision") + **kwargs: Additional keyword arguments passed to the base class + + Supported Image Formats: + - JPEG/JPG + - PNG + - GIF + - WEBP + + Image Input Types: + - URL to image (str) + - File path (str) + - Base64 encoded string + - Bytes object + - PIL Image object + - NumPy array + + Usage: + ```python + # Initialize the model + model = Gemini_With_Vision("gemini-1.5-flash") + + # Text-only generation + response = model.generate_response({ + "System": "You are a helpful assistant.", + "User": "What is the capital of France?" + }) + + # Image from URL + response = model.generate_response({ + "System": "Analyze this image.", + "User": "What do you see?" + }, image_parts="https://example.com/image.jpg") + + # Image from local file + from PIL import Image + image = Image.open("path/to/image.jpg") + + response = model.generate_response({ + "System": "You are a helpful assistant that can analyze images.", + "User": "What do you see in this image?" + }, image_parts=image) + + # Multiple images + response = model.generate_response({ + "System": "Compare these images.", + "User": "What are the differences?" + }, image_parts=[image1, image2]) + ``` + + Notes: + - Maximum image size: 20MB + - For optimal performance, ensure images are in supported formats + - When using multiple images, provide them as a list in image_parts + - URLs must start with 'http://' or 'https://' and have a 10-second timeout + """ + + def __init__(self, model_name, **kwargs): + super().__init__(model_name, **kwargs) + self._model = genai.GenerativeModel(model_name) + + @staticmethod + def _prepare_prompt(model_prompt): + return '\n\n'.join([model_prompt.get('system', ''), model_prompt.get('user', '')]) + + def _process_image_input(self, image_input: Union[str, bytes, Image.Image, np.ndarray]) -> Any: + """ + Process different types of image inputs into a format acceptable by Gemini. + + Parameters: + image_input: Can be: + - URL to image (str) + - Path to image file (str) + - Base64 encoded image string + - Bytes object + - PIL Image object + - NumPy array + + Returns: + Processed image in a format acceptable by Gemini + """ + try: + # Verify file type and size before processing + def verify_image(img: Image.Image) -> bool: + """Verify image format and size""" + allowed_formats = {'JPEG', 'JPG', 'PNG', 'GIF', 'WEBP'} + max_size = 20 * 1024 * 1024 # 20MB limit + + # Check format + if img.format and img.format.upper() not in allowed_formats: + self.logger.log(f"Unsupported image format: {img.format}", 'warning') + return False + + # Check file size + img_byte_arr = BytesIO() + img.save(img_byte_arr, format=img.format or 'PNG') + size = img_byte_arr.tell() + if size > max_size: + self.logger.log(f"Image size ({size/1024/1024:.2f}MB) exceeds 20MB limit", 'warning') + return False + + return True + + self.logger.log(f"Processing image input of type: {type(image_input)}", 'info') + + if isinstance(image_input, str): + # Check if it's a URL + if image_input.startswith(('http://', 'https://')): + self.logger.log(f"Downloading image from URL: {image_input}", 'info') + response = requests.get(image_input, timeout=10) + response.raise_for_status() # Raise exception for bad status codes + img = Image.open(BytesIO(response.content)) + # Check if it's a base64 string + elif image_input.startswith(('data:image', 'base64:')): + self.logger.log("Processing base64 encoded image", 'info') + # Extract the base64 data + base64_data = image_input.split('base64,')[-1] + image_bytes = base64.b64decode(base64_data) + img = Image.open(BytesIO(image_bytes)) + else: + # Assume it's a file path + self.logger.log(f"Loading image from path: {image_input}", 'info') + img = Image.open(image_input) + + elif isinstance(image_input, bytes): + self.logger.log("Processing bytes input", 'info') + img = Image.open(BytesIO(image_input)) + + elif isinstance(image_input, np.ndarray): + self.logger.log("Processing NumPy array input", 'info') + img = Image.fromarray(image_input) + + elif isinstance(image_input, Image.Image): + self.logger.log("Processing PIL Image input", 'info') + img = image_input + + else: + error_msg = f"Unsupported image input type: {type(image_input)}" + self.logger.log(error_msg, 'error') + raise ValueError(error_msg) + + # Verify the processed image + if not verify_image(img): + error_msg = "Image verification failed" + self.logger.log(error_msg, 'error') + raise ValueError(error_msg) + + self.logger.log(f"Successfully processed image: {img.format} {img.size}", 'info') + return img + + except Exception as e: + error_msg = f"Error processing image input: {str(e)}" + self.logger.log(error_msg, 'error') + raise + + def _do_api_call(self, prompt, **filtered_params): + # Combine text prompt with images if provided + content = [prompt] + image_parts = filtered_params.pop('images', None) + + if image_parts is not None: + # Convert single image to list for uniform processing + if not isinstance(image_parts, list): + image_parts = [image_parts] + + # Process each image input + processed_images = [] + for img in image_parts: + try: + processed_img = self._process_image_input(img) + processed_images.append(processed_img) + except Exception as e: + self.logger.log(f"Error processing image: {str(e)}", 'error') + continue + + content.extend(processed_images) + + response = self._model.generate_content( + content, + safety_settings={ + HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE, + HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE, + HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE, + HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE, + }, + generation_config=genai.types.GenerationConfig( + max_output_tokens=filtered_params.get("max_new_tokens", 2048), + temperature=filtered_params.get("temperature", 0.7), + top_p=filtered_params.get("top_p", 1), + top_k=filtered_params.get("top_k", 1), + candidate_count=max(filtered_params.get("candidate_count", 1), 1) + ) + ) + return response + + def _process_response(self, raw_response): + return raw_response.text + + def generate_response(self, model_prompt, image_parts=None, **params): + """Wrapper method to maintain backward compatibility""" + return self.generate(model_prompt, image_parts=image_parts, **params) + + +if __name__ == "__main__": + # Initialize the model + gemini_vision = Gemini_With_Vision("gemini-1.5-flash") + + # Test 1: Text-only input + print("\n=== Test 1: Text-only Input ===") + text_response = gemini_vision.generate_response({ + "System": "You are a helpful assistant.", + "User": "What is the capital of France?" + }, agent_name="TestAgent") + print("Response:", text_response) + + # Test 2: Direct URL input + print("\n=== Test 2: URL Input ===") + try: + image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/4/4b/La_Tour_Eiffel_vue_de_la_Tour_Saint-Jacques%2C_Paris_ao%C3%BBt_2014_%282%29.jpg/1200px-La_Tour_Eiffel_vue_de_la_Tour_Saint-Jacques%2C_Paris_ao%C3%BBt_2014_%282%29.jpg" + url_response = gemini_vision.generate_response({ + "System": "You are a helpful assistant that can analyze images.", + "User": "What do you see in this image? Please describe it in detail." + }, image_parts=image_url, agent_name="TestAgent") + print("Response:", url_response) + except Exception as e: + print(f"Error during URL test: {str(e)}") + + # Test 3: PIL Image input + print("\n=== Test 3: PIL Image Input ===") + try: + from PIL import Image + import requests + from io import BytesIO + + # Download and convert to PIL Image + response = requests.get(image_url) + image = Image.open(BytesIO(response.content)) + + vision_response = gemini_vision.generate_response({ + "System": "You are a helpful assistant that can analyze images.", + "User": "What do you see in this image? Please describe it in detail." + }, image_parts=image, agent_name="TestAgent") + print("Response:", vision_response) + except Exception as e: + print(f"Error during PIL image test: {str(e)}") + + # Test 4: Base64 image input + print("\n=== Test 4: Base64 Input ===") + try: + buffered = BytesIO() + image.save(buffered, format="JPEG") + img_str = base64.b64encode(buffered.getvalue()).decode() + base64_response = gemini_vision.generate_response({ + "System": "You are a helpful assistant that can analyze images.", + "User": "What do you see in this image? Please describe it in detail." + }, image_parts=f"data:image/jpeg;base64,{img_str}", agent_name="TestAgent") + print("Response:", base64_response) + except Exception as e: + print(f"Error during base64 image test: {str(e)}") + + # Test 5: NumPy array input + print("\n=== Test 5: NumPy Array Input ===") + try: + import numpy as np + np_image = np.array(image) + numpy_response = gemini_vision.generate_response({ + "System": "You are a helpful assistant that can analyze images.", + "User": "What do you see in this image? Please describe it in detail." + }, image_parts=np_image, agent_name="TestAgent") + print("Response:", numpy_response) + except Exception as e: + print(f"Error during NumPy array test: {str(e)}") + + # Test 6: Format verification + print("\n=== Test 6: Format Verification ===") + try: + temp_image = Image.new('RGB', (100, 100), color='red') + buffered = BytesIO() + temp_image.save(buffered, format="BMP") + bmp_data = buffered.getvalue() + + bmp_response = gemini_vision.generate_response({ + "System": "You are a helpful assistant that can analyze images.", + "User": "What do you see in this image? Please describe it in detail." + }, image_parts=bmp_data, agent_name="TestAgent") + print("Response:", bmp_response) + except Exception as e: + print(f"Expected error during BMP image test: {str(e)}") \ No newline at end of file diff --git a/src/agentforge/apis/groq_api.py b/src/agentforge/apis/groq_api.py new file mode 100644 index 00000000..5986cfa1 --- /dev/null +++ b/src/agentforge/apis/groq_api.py @@ -0,0 +1,20 @@ +import os +from .base_api import BaseModel +from groq import Groq +# from agentforge.utils.Logger import Logger + +api_key = os.getenv("GROQ_API_KEY") +client = Groq(api_key=api_key) + +class GroqAPI(BaseModel): + + def _do_api_call(self, prompt, **filtered_params): + response = client.chat.completions.create( + model=self.model_name, + messages=prompt, + **filtered_params + ) + return response + + def _process_response(self, raw_response): + return raw_response.choices[0].message.content diff --git a/src/agentforge/apis/lm_studio_api.py b/src/agentforge/apis/lm_studio_api.py new file mode 100644 index 00000000..e01726a3 --- /dev/null +++ b/src/agentforge/apis/lm_studio_api.py @@ -0,0 +1,29 @@ +import requests +import json +from .base_api import BaseModel + +class LMStudio(BaseModel): + """ + Concrete implementation for OpenAI GPT models. + """ + + def _do_api_call(self, prompt, **filtered_params): + url = filtered_params.pop('host_url', 'http://localhost:1234/v1/chat/completions') + headers = {'Content-Type': 'application/json'} + data = { + "model": self.model_name, + "messages": prompt, + **filtered_params + } + + response = requests.post(url, headers=headers, json=data) + + if response.status_code != 200: + # return error content + self.logger.log(f"Request error: {response}", 'error') + return None + + return response.json() + + def _process_response(self, raw_response): + return raw_response["choices"][0]["message"]["content"] diff --git a/src/agentforge/apis/ollama_api.py b/src/agentforge/apis/ollama_api.py new file mode 100755 index 00000000..ceb67241 --- /dev/null +++ b/src/agentforge/apis/ollama_api.py @@ -0,0 +1,31 @@ +import requests +import json +from .base_api import BaseModel + +class Ollama(BaseModel): + + @staticmethod + def _prepare_prompt(model_prompt): + return model_prompt + + def _do_api_call(self, prompt, **filtered_params): + url = filtered_params.pop('host_url', 'http://localhost:11434/api/generate') + headers = {'Content-Type': 'application/json'} + data = { + "model": self.model_name, + "system": prompt.get('system'), + "prompt": prompt.get('user'), + **filtered_params + } + + response = requests.post(url, headers=headers, json=data) + + if response.status_code != 200: + # return error content + self.logger.log(f"Request error: {response}", 'error') + return None + + return response.json() + + def _process_response(self, raw_response): + return raw_response['choices'][0]['message']['content'] diff --git a/src/agentforge/apis/openai_api.py b/src/agentforge/apis/openai_api.py new file mode 100755 index 00000000..5f722662 --- /dev/null +++ b/src/agentforge/apis/openai_api.py @@ -0,0 +1,32 @@ +from .base_api import BaseModel +from openai import OpenAI + +# Assuming you have set OPENAI_API_KEY in your environment variables +client = OpenAI() + +class GPT(BaseModel): + """ + Concrete implementation for OpenAI GPT models. + """ + + def _do_api_call(self, prompt, **filtered_params): + response = client.chat.completions.create( + model=self.model_name, + messages=prompt, + **filtered_params + ) + return response + + def _process_response(self, raw_response): + return raw_response.choices[0].message.content + +class O1Series(GPT): + """ + Concrete implementation for OpenAI GPT models. + """ + + def _prepare_prompt(self, model_prompt): + # Format user messages in the appropriate style + content = f"{model_prompt.get('system', '')}\n\n{model_prompt.get('user', '')}" + return [{"role": "user", "content": content}] + diff --git a/src/agentforge/apis/openrouter_api.py b/src/agentforge/apis/openrouter_api.py new file mode 100644 index 00000000..71fe06a5 --- /dev/null +++ b/src/agentforge/apis/openrouter_api.py @@ -0,0 +1,43 @@ +import os +import time +import requests +from .base_api import BaseModel +from agentforge.utils.logger import Logger + +# Get the API key from the environment variable +api_key = os.getenv('OPENROUTER_API_KEY') + + +class OpenRouter(BaseModel): + """ + A class for interacting with OpenRouter's API to generate text based on provided prompts. + + Handles API calls to OpenRouter, including error handling and retries for failed requests. + """ + + + def _do_api_call(self, prompt, **filtered_params): + url = filtered_params.pop('host_url', 'https://openrouter.ai/api/v1/chat/completions') + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + "HTTP-Referer": filtered_params.pop("http_referer", ""), + "X-Title": 'AgentForge' + } + data = { + "model": self.model_name, + "messages": prompt, + **filtered_params + } + + response = requests.post(url, headers=headers, json=data) + + if response.status_code != 200: + # return error content + self.logger.log(f"Request error: {response}", 'error') + return None + + return response.json() + + def _process_response(self, raw_response): + return raw_response['choices'][0]['message']['content'] diff --git a/src/agentforge/cogarch.py b/src/agentforge/cogarch.py new file mode 100755 index 00000000..ae07b246 --- /dev/null +++ b/src/agentforge/cogarch.py @@ -0,0 +1,342 @@ +from agentforge.config import Config +from agentforge.utils.logger import Logger +import importlib + + +class CogArch: + def __init__(self, name): + """ + Initializes a CogArch instance with the given name. + """ + self.name = name + self.logger = Logger(name=self.name) + self.config = Config() + self.agents = {} + self.context = {} + self.flow = {} + self.step_dict = {} + + self.load_flow() + self.load_agents() + self.initialize_context() + self.validate_flow() + + def load_flow(self): + """ + Loads the flow configuration for the cognitive architecture. + """ + try: + self.flow = self.config.load_flow_data(self.name) + except Exception as e: + self.logger.log(f"Error loading flow '{self.name}': {e}", level='error') + raise + + def initialize_context(self): + """ + Placeholder for initializing the context. Can be overridden by subclasses. + """ + pass + + def load_agents(self): + """ + Loads and initializes agents as defined in the flow configuration. + """ + agents_config = self.flow.get('agents', []) + if not agents_config: + error_msg = f"No agents defined in cognitive architecture '{self.name}'." + self.logger.log(error_msg, level='error') + raise ValueError(error_msg) + + for agent_info in agents_config: + agent_name = self.get_agent_name(agent_info) + agent_class = self.get_agent_class(agent_info, agent_name) + agent_instance = self.instantiate_agent(agent_class, agent_name) + self.agents[agent_name] = agent_instance + + def get_agent_name(self, agent_info): + """ + Extracts and returns the agent name from the agent configuration. + """ + agent_name = agent_info.get('name') + if not agent_name: + error_msg = f"Agent name missing in cognitive architecture '{self.name}'." + self.logger.log(error_msg, level='error') + raise ValueError(error_msg) + return agent_name + + def get_agent_class(self, agent_info, agent_name): + """ + Imports and returns the agent class based on the provided class path. + Defaults to the base Agent class if no class path is provided. + """ + module_path = 'agentforge.agent' + class_name = 'Agent' + + class_path = agent_info.get('class', '') + if class_path: + try: + # module_path, class_name = class_path.rsplit('.', 1) + module_path = class_path + _, class_name = class_path.rsplit('.', 1) + except Exception as e: + error_msg = f"Error parsing class path '{class_path}' for agent '{agent_name}': {e}" + self.logger.log(error_msg, level='error') + raise ValueError(error_msg) + + try: + module = importlib.import_module(module_path) + agent_class = getattr(module, class_name) + self.logger.log(f"Loaded '{class_name}' from '{module_path}' for agent '{agent_name}'.", level='info') + return agent_class + except Exception as e: + error_msg = f"Error importing agent class '{class_name}' from '{module_path}' for agent '{agent_name}': {e}" + self.logger.log(error_msg, level='error') + raise ImportError(error_msg) + + def instantiate_agent(self, agent_class, agent_name): + """ + Instantiates and returns an agent instance. + """ + try: + agent_instance = agent_class(agent_name=agent_name) + self.logger.log(f"Agent '{agent_name}' instantiated.", level='info') + return agent_instance + except Exception as e: + error_msg = f"Error instantiating agent '{agent_name}': {e}" + self.logger.log(error_msg, level='error') + raise RuntimeError(error_msg) + + def validate_flow(self): + """ + Validates the flow configuration to ensure all agents and steps are properly defined. + """ + steps = self.flow.get('flow', []) + if not steps: + error_msg = f"No flow steps defined for cognitive architecture '{self.name}'." + self.logger.log(error_msg, level='error') + raise ValueError(error_msg) + + # Map agent names to steps for quick lookup + self.step_dict = {} + for step in steps: + self.validate_step_structure(step) + agent_name = step['step']['agent'] + self.check_duplicate_step(agent_name) + self.step_dict[agent_name] = step + self.validate_agent_defined(agent_name) + self.validate_next_agents(step) + self.validate_condition_agents(step) + + self.logger.log("Flow validation completed successfully.", level='info') + + def validate_step_structure(self, step): + """ + Validates that each step has the required structure. + """ + if 'step' not in step or 'agent' not in step['step']: + error_msg = "Each flow step must contain a 'step' dictionary with an 'agent' key." + self.logger.log(error_msg, level='error') + raise ValueError(error_msg) + + def check_duplicate_step(self, agent_name): + """ + Checks for duplicate steps for the same agent. + """ + if agent_name in self.step_dict: + error_msg = f"Duplicate step for agent '{agent_name}'." + self.logger.log(error_msg, level='error') + raise ValueError(error_msg) + + def validate_agent_defined(self, agent_name): + """ + Validates that the agent is defined in the agents list. + """ + if agent_name not in self.agents: + error_msg = f"Agent '{agent_name}' in flow is not defined in agents list." + self.logger.log(error_msg, level='error') + raise ValueError(error_msg) + + def validate_next_agents(self, step): + """ + Validates that all 'next' agents are defined. + """ + step_info = step['step'] + next_agents = step_info.get('next', []) + if isinstance(next_agents, str): + next_agents = [next_agents] + for next_agent in next_agents: + if next_agent not in self.agents: + error_msg = f"Next agent '{next_agent}' in step '{step_info['agent']}' is not defined." + self.logger.log(error_msg, level='error') + raise ValueError(error_msg) + + def validate_condition_agents(self, step): + """ + Validates that all agents referenced in conditions are defined. + """ + step_info = step['step'] + condition = step_info.get('condition') + if not condition: + return + + condition_type = condition.get('type') + if condition_type not in ('expression', 'function', 'variable'): + error_msg = f"Unsupported condition type '{condition_type}' in agent '{step_info['agent']}'." + self.logger.log(error_msg, level='error') + raise ValueError(error_msg) + + # Collect all possible next agents from conditions + condition_agents = [] + if condition_type == 'variable': + cases = condition.get('cases', {}) + default_agent = condition.get('default') + condition_agents.extend(cases.values()) + if default_agent: + condition_agents.append(default_agent) + else: + condition_agents.extend([ + condition.get('on_true'), + condition.get('on_false') + ]) + + for cond_agent in condition_agents: + if cond_agent and cond_agent not in self.agents: + error_msg = f"Condition agent '{cond_agent}' in step '{step_info['agent']}' is not defined." + self.logger.log(error_msg, level='error') + raise ValueError(error_msg) + + def run(self, **kwargs): + """ + Executes the cognitive architecture's flow starting from the first step. + """ + self.context.update(kwargs) + + # Start execution from the first step + steps = self.flow.get('flow', []) + current_step = steps[0] + + while current_step: + agent_name = current_step['step']['agent'] + self.execute_step(agent_name) + + next_step = self.get_next_step(current_step) + if next_step: + current_step = next_step + continue + self.logger.log(f"Flow completed at agent '{agent_name}'.", level='info') + current_step = None # End the loop + + def execute_step(self, agent_name): + """ + Executes an agent and updates the context with its output. + """ + agent_instance = self.agents[agent_name] + self.logger.log(f"Running agent '{agent_name}'.", level='info') + agent_input = self.prepare_input(agent_name) + agent_output = self.run_agent(agent_instance, agent_input) + self.context[agent_name] = agent_output + + def prepare_input(self, agent_name): + """ + Prepares the input data for the agent. Can be overridden for custom behavior. + """ + return self.context + + def run_agent(self, agent_instance, agent_input): + """ + Runs the agent with the provided input. + """ + try: + agent_output = agent_instance.run(**agent_input) + return agent_output + except Exception as e: + error_msg = f"Error running agent '{agent_instance.agent_name}': {e}" + self.logger.log(error_msg, level='error') + raise RuntimeError(error_msg) + + def get_next_step(self, current_step): + """ + Determines the next step based on 'condition' or 'next' in current_step. + Returns the next step to process or None if there is no next step. + """ + step_info = current_step['step'] + if 'condition' in step_info: + next_agent_name = self.evaluate_condition(step_info['condition']) + next_step = self.step_dict.get(next_agent_name) + return next_step + + if 'next' in step_info: + next_agents = step_info.get('next', []) + if isinstance(next_agents, str): + next_agents = [next_agents] + next_agent_name = next_agents[0] if next_agents else None + if next_agent_name: + next_step = self.step_dict.get(next_agent_name) + return next_step + + # No 'next' or 'condition' specified + return None + + def evaluate_condition(self, condition): + """ + Evaluates a condition and determines the next agent based on the result. + """ + condition_type = condition.get('type') + if condition_type == 'expression': + return self.evaluate_expression_condition(condition) + if condition_type == 'function': + return self.evaluate_function_condition(condition) + if condition_type == 'variable': + return self.evaluate_variable_condition(condition) + + error_msg = f"Unsupported condition type '{condition_type}'." + self.logger.log(error_msg, level='error') + raise ValueError(error_msg) + + def evaluate_expression_condition(self, condition): + """ + Evaluates an expression condition. + """ + expression = condition.get('expression') + result = self.evaluate_expression(expression) + return condition.get('on_true') if result else condition.get('on_false') + + def evaluate_function_condition(self, condition): + """ + Evaluates a function-based condition. + """ + function_name = condition.get('function') + function = getattr(self, function_name, None) + if not function: + error_msg = f"Condition function '{function_name}' not found." + self.logger.log(error_msg, level='error') + raise ValueError(error_msg) + + result = function() + return condition.get('on_true') if result else condition.get('on_false') + + def evaluate_variable_condition(self, condition): + """ + Evaluates a condition based on the value of a variable in the context. + """ + variable_name = condition.get('on') + variable_value = self.context.get(variable_name) + cases = condition.get('cases', {}) + default = condition.get('default') + return cases.get(variable_value, default) + + def evaluate_expression(self, expression): + """ + Safely evaluates an expression using the context. + """ + try: + allowed_names = {} + eval_context = {} + eval_context.update(allowed_names) + eval_context.update(self.context) + value = eval(expression, {"__builtins__": None}, eval_context) + return value + except Exception as e: + error_msg = f"Error evaluating expression '{expression}': {e}" + self.logger.log(error_msg, level='error') + raise RuntimeError(error_msg) diff --git a/src/agentforge/config.py b/src/agentforge/config.py index 7cb501e6..9ffade62 100755 --- a/src/agentforge/config.py +++ b/src/agentforge/config.py @@ -1,32 +1,23 @@ import importlib +import threading import os import yaml import re import pathlib import sys -from typing import Dict, Any +from typing import Dict, Any, Optional, Tuple +from ruamel.yaml import YAML -def load_yaml_file(file_path: str): +def load_yaml_file(file_path: str) -> Dict[str, Any]: """ Reads and parses a YAML file, returning its contents as a Python dictionary. - This function attempts to safely load the contents of a YAML file specified by - the file path. If the file cannot be found or there's an error decoding the YAML, - it handles the exceptions gracefully. - Parameters: file_path (str): The path to the YAML file to be read. Returns: dict: The contents of the YAML file as a dictionary. If the file is not found or an error occurs during parsing, an empty dictionary is returned. - - Notes: - - The function uses `yaml.safe_load` to prevent execution of arbitrary code - that might be present in the YAML file. - - Exceptions for file not found and YAML parsing errors are caught and logged, - with an empty dictionary returned to allow the calling code to continue - execution without interruption. """ try: with open(file_path, 'r') as yaml_file: @@ -38,11 +29,16 @@ def load_yaml_file(file_path: str): print(f"Error decoding YAML from {file_path}") return {} - class Config: + _debug = False _instance = None + _lock = threading.Lock() # Class-level lock for thread safety pattern = r"^[a-zA-Z_][a-zA-Z0-9_]*$" + # ------------------------------------------------------------------------ + # Initialization + # ------------------------------------------------------------------------ + def __new__(cls, *args, **kwargs): """ Ensures that only one instance of Config exists. @@ -51,289 +47,342 @@ def __new__(cls, *args, **kwargs): Returns: Config: The singleton instance of the Config class. """ - if not cls._instance: - cls._instance = super(Config, cls).__new__(cls, *args, **kwargs) + with cls._lock: + if not cls._instance: + cls._instance = super(Config, cls).__new__(cls) return cls._instance - def __init__(self): + def __init__(self, root_path: Optional[str] = None): """ Initializes the Config object, setting up the project root and configuration path. Calls method to load configuration data from YAML files. """ if not hasattr(self, 'is_initialized'): # Prevent re-initialization self.is_initialized = True - try: - self.project_root = self.find_project_root() - self.config_path = self.project_root / ".agentforge" - # Placeholders for the data the agent needs which is located in each respective YAML file - self.data = {} + self.project_root = self.find_project_root(root_path) + self.config_path = self.project_root / ".agentforge" - # Here is where we load the information from the YAML files to their corresponding attributes - self.load_all_configurations() - except Exception as e: - raise ValueError(f"Error during Config initialization: {e}") + # Placeholder for configuration data loaded from YAML files + self.data = {} - @staticmethod - def find_project_root(): - """ - Finds the project root by searching for the .agentforge directory. - - Returns: - pathlib.Path: The path to the project root directory. + # Load the configuration data + self.load_all_configurations() - Raises: - FileNotFoundError: If the .agentforge directory cannot be found. + @classmethod + def reset(cls, root_path=None): """ - print(f"\n\nCurrent working directory: {os.getcwd()}") - + Completely resets the Config singleton, allowing for re-initialization. + """ + cls._instance = None + return cls(root_path=root_path) + + def find_project_root(self, root_path: Optional[str] = None) -> pathlib.Path: + # If a root path was provided, use it to checking that .agentforge exists + if root_path: + custom_root = pathlib.Path(root_path).resolve() + agentforge_dir = custom_root / ".agentforge" + if agentforge_dir.is_dir(): + if self._debug: print(f"\n\nUsing custom project root: {custom_root}") + return custom_root + # Early return or raise an error if .agentforge isn’t found in the custom path + raise FileNotFoundError(f"No .agentforge found in custom root path: {custom_root}") + + # Otherwise, fall back to the original search logic script_dir = pathlib.Path(sys.argv[0]).resolve().parent current_dir = script_dir + if self._debug: print(f"\n\nCurrent working directory: {os.getcwd()}") while current_dir != current_dir.parent: potential_dir = current_dir / ".agentforge" - print(f"Checking {potential_dir}") # Debugging output + if self._debug: print(f"Checking {potential_dir}") if potential_dir.is_dir(): - print(f"Found .agentforge directory at: {current_dir}") # Debugging output + if self._debug: print(f"Found .agentforge directory at: {current_dir}\n") return current_dir - current_dir = current_dir.parent - raise FileNotFoundError(f"Could not find the '.agentforge' directory at {script_dir}") - - @staticmethod - def get_nested_dict(data: dict, path_parts: tuple): - """ - Gets or creates a nested dictionary given the parts of a relative path. - - Args: - data (dict): The top-level dictionary to start from. - path_parts (tuple): A tuple of path components leading to the desired nested dictionary. - - Returns: - A reference to the nested dictionary at the end of the path. - """ - for part in path_parts: - if part not in data: - data[part] = {} - data = data[part] - return data - - def find_agent_config(self, agent_name: str): - """ - Search for an agent's configuration by name within the nested agents' dictionary. - - Parameters: - agent_name (str): The name of the agent to find. - - Returns: - dict: The configuration dictionary for the specified agent, or None if not found. - """ - - def search_nested_dict(nested_dict, target): - for key, value in nested_dict.items(): - if key == target: - return value - elif isinstance(value, dict): - result = search_nested_dict(value, target) - if result is not None: - return result - return None + raise FileNotFoundError(f"Could not find the '.agentforge' directory starting from {script_dir}") - return search_nested_dict(self.data.get('prompts', {}), agent_name) + # ------------------------------------------------------------------------ + # Configuration Loading + # ------------------------------------------------------------------------ def load_all_configurations(self): """ Recursively loads all configuration data from YAML files under each subdirectory of the .agentforge folder. """ - for subdir, dirs, files in os.walk(self.config_path): - for file in files: - if file.endswith(('.yaml', '.yml')): - subdir_path = pathlib.Path(subdir) - relative_path = subdir_path.relative_to(self.config_path) - nested_dict = self.get_nested_dict(self.data, relative_path.parts) - - file_path = str(subdir_path / file) - data = load_yaml_file(file_path) - if data: - filename_without_ext = os.path.splitext(file)[0] - nested_dict[filename_without_ext] = data - - def find_file_in_directory(self, directory: str, filename: str): + with self._lock: + for subdir, dirs, files in os.walk(self.config_path): + for file in files: + if file.endswith(('.yaml', '.yml')): + subdir_path = pathlib.Path(subdir) + relative_path = subdir_path.relative_to(self.config_path) + nested_dict = self.get_nested_dict(self.data, relative_path.parts) + + file_path = str(subdir_path / file) + data = load_yaml_file(file_path) + if data: + filename_without_ext = os.path.splitext(file)[0] + nested_dict[filename_without_ext] = data + + # ------------------------------------------------------------------------ + # Save Configuration Method + # ------------------------------------------------------------------------ + + def save(self): """ - Recursively searches for a file within a directory and its subdirectories. - - Parameters: - directory (str): The directory to search in. - filename (str): The name of the file to find. - - Returns: - pathlib.Path or None: The full path to the file if found, None otherwise. + Saves changes to the configuration back to the system.yaml file, + preserving structure, formatting, and comments. """ - directory = pathlib.Path(self.get_file_path(directory)) + with Config._lock: + system_yaml_path = self.config_path / 'settings' / 'system.yaml' - for file_path in directory.rglob(filename): - return file_path - return None + _yaml = YAML() + _yaml.preserve_quotes = True # Preserves quotes around strings if they exist - def get_file_path(self, file_name: str): - """ - Constructs the full path for a given filename within the configuration path. + try: + # Load the existing system.yaml with formatting preserved + with open(system_yaml_path, 'r') as yaml_file: + existing_data = _yaml.load(yaml_file) + + if 'settings' in self.data and 'system' in self.data['settings']: + # Update the existing data with the new settings + for key, value in self.data['settings']['system'].items(): + if isinstance(value, dict) and key in existing_data: + # Merge dictionaries for nested structures + existing_data[key].update(value) + continue + + # Replace or add the top-level key + existing_data[key] = value + + # Save the updated configuration back to the file + with open(system_yaml_path, 'w') as yaml_file: + _yaml.dump(existing_data, yaml_file) + return + print("No system settings to save.") + except Exception as e: + print(f"Error saving configuration to {system_yaml_path}: {e}") - Parameters: - file_name (str): The name of the file. + # ------------------------------------------------------------------------ + # Agent and Flow Configuration + # ------------------------------------------------------------------------ - Returns: - pathlib.Path: The full path to the file within the configuration directory. + def load_agent_data(self, agent_name: str) -> Dict[str, Any]: + """ + Loads configuration data for a specified agent, applying any overrides in the agent’s config. + Returns a dict containing everything needed to run that agent. """ - return pathlib.Path(self.config_path) / file_name - def get_llm(self, api: str, model: str): + agent = self.find_config('prompts', agent_name) + + api_name, class_name, model_name, final_params = self.resolve_model_overrides(agent) + model = self.get_model(api_name, class_name, model_name) + + persona_data = self.load_persona(agent) + prompts = self.fix_prompt_placeholders(agent.get('prompts', {})) + settings = self.data.get('settings', {}) + + default_debug_text = settings['system']['debug'].get('simulated_response', 'Simulated Text Goes Here!!!') + simulated_response = agent.get('simulated_response', default_debug_text).strip() + + return { + 'name': agent_name, + 'settings': settings, + 'model': model, + 'params': final_params, + 'persona': persona_data, + 'prompts': prompts, + 'simulated_response': simulated_response, + } + + # def load_flow_data(self, flow_name: str) -> Dict[str, Any]: + # """ + # Loads configuration data for a specified flow. + # + # Parameters: + # flow_name (str): The name of the flow to load. + # + # Returns: + # dict: The configuration data for the flow. + # + # Raises: + # FileNotFoundError: If the flow configuration is not found. + # """ + # self.reload() + # + # flow = self.find_config('flows', flow_name) + # if not flow: + # raise FileNotFoundError(f"Flow '{flow_name}' not found in configuration.") + # + # return flow + + # ------------------------------------------------------------------------ + # Model Overrides + # ------------------------------------------------------------------------ + + def resolve_model_overrides(self, agent: dict) -> Tuple[str, str, str, Dict[str, Any]]: + """ + Orchestrates finding and merging all relevant model overrides into + a final 4-tuple: (api_name, class_name, model_identifier, final_params). """ - Loads a specified language model based on API and model settings. + api_name, model_name, agent_params_override = self._get_agent_api_and_model(agent) + api_section = self._get_api_section(api_name) + class_name, model_data = self._find_class_for_model(api_section, model_name) - Parameters: - api (str): The API name. - model (str): The model name. + model_identifier = self._get_model_identifier(api_name, model_name, model_data) + final_params = self._merge_params(api_section, class_name, model_data, agent_params_override) - Returns: - object: An instance of the requested model class. + return api_name, class_name, model_identifier, final_params - Raises: - Exception: If there is an error loading the model. + # Step 1: Identify which API and model is needed, returning agent-level params + def _get_agent_api_and_model(self, agent: dict) -> Tuple[str, str, Dict[str, Any]]: + """ + Reads the 'Selected Model' defaults from the YAML and merges any agent-level + overrides, returning (api_name, model_name, agent_params_override). + Raises a ValueError if no valid API/Model can be determined. """ - try: - # Retrieve the model name, module, and class from the 'models.yaml' settings. - model_name = self.data['settings']['models']['ModelLibrary'][api]['models'][model]['name'] - module_name = self.data['settings']['models']['ModelLibrary'][api]['module'] - class_name = self.data['settings']['models']['ModelLibrary'][api]['class'] + selected_model = self.data['settings']['models'].get('default_model', {}) + default_api = selected_model.get('api') + default_model = selected_model.get('model') - # Dynamically import the module corresponding to the LLM API. - module = importlib.import_module(f".llm.{module_name}", package=__package__) + model_overrides = agent.get('model_overrides', {}) + api_name = model_overrides.get('api', default_api) + model_name = model_overrides.get('model', default_model) + agent_params_override = model_overrides.get('params', {}) - # Retrieve the class from the imported module that handles the LLM connection. - model_class = getattr(module, class_name) - model_class = getattr(module, class_name) - args = [model_name] # Prepare the arguments for the model class instantiation. - return model_class(*args) # Instantiate the model class with the provided arguments. + if not api_name or not model_name: + raise ValueError("No valid API/Model found in either Selected Model defaults or agent overrides.") + return api_name, model_name, agent_params_override - except Exception as e: - print(f"Error Loading Model: {e}") - raise - def load_agent(self, agent_name: str): + # Step 2: Retrieve the correct API section from the Model Library + def _get_api_section(self, api_name: str) -> Dict[str, Any]: """ - Loads an agent's configuration from a YAML file. - - Parameters: - agent_name (str): The name of the agent to load. - - Raises: - FileNotFoundError: If the agent's YAML file cannot be found. - Exception: For any errors encountered while loading the agent. + Returns the relevant subsection of the Model Library for the requested API. + Raises a ValueError if the API is not in the Model Library. """ - try: - path_to_file = self.find_file_in_directory("prompts", f"{agent_name}.yaml") - if path_to_file: - self.data['agent'] = load_yaml_file(str(path_to_file)) # Fix warning - else: - raise FileNotFoundError(f"Agent {agent_name}.yaml not found.") - except Exception as e: - print(f"Error loading agent {agent_name}: {e}") + model_library = self.data['settings']['models'].get('model_library', {}) + if api_name not in model_library: + raise ValueError(f"API '{api_name}' does not exist in the Model Library.") + return model_library[api_name] - def load_persona(self, agent: dict) -> dict | None: + # Step 3: Locate which class has the requested model and return its data + @staticmethod + def _find_class_for_model(api_section: Dict[str, Any], model_name: str) -> Tuple[str, Dict[str, Any]]: """ - Loads the persona for the agent, if personas are enabled. - - Parameters: - agent (dict): The agent's configuration data. - - Returns: - dict: The loaded persona and the persona file name. + Loops over the classes in the given API section to find one that has 'model_name' + in its 'models' dict. Returns (class_name, model_data). + Raises a ValueError if the model is not found. """ - persona_data = None - - settings = self.data['settings'] - if not settings['system'].get('PersonasEnabled', False): - return persona_data - - persona_file = agent.get('Persona') or settings['system'].get('Persona', 'default') - if persona_file not in self.data['personas']: - raise FileNotFoundError(f"Selected Persona '{persona_file}' not found. Please make sure the corresponding persona file is in the personas folder") + for candidate_class, class_config in api_section.items(): + if candidate_class == 'params': # skip any top-level 'params' key + continue + models_dict = class_config.get('models', {}) + if model_name in models_dict: + return candidate_class, models_dict[model_name] - persona_data = self.data['personas'][persona_file] - return persona_data + raise ValueError(f"Model '{model_name}' not found in this API section.") - def load_agent_data(self, agent_name: str) -> Dict[str, Any]: + # Step 4: Get the unique identifier for the model + @staticmethod + def _get_model_identifier(api_name: str, model_name: str, model_data: Dict[str, Any]) -> str: + """ + Reads the 'identifier' for the selected model from the YAML. + Raises a ValueError if it doesn't exist. """ - Loads configuration data for a specified agent, applying any overrides specified in the agent's configuration. + identifier = model_data.get('identifier') + if not identifier: + raise ValueError(f"Identifier not found for Model '{model_name}' under API '{api_name}' in Model Library.") + return identifier - Parameters: - agent_name (str): The name of the agent for which to load configuration data. + # Step 5: Merge API-level, class-level, model-level params, and agent overrides + @staticmethod + def _merge_params(api_section: Dict[str, Any], class_name: str, model_data: Dict[str, Any], + agent_params_override: Dict[str, Any]) -> Dict[str, Any]: + """ + Takes all the relevant parameter dicts and merges them in ascending specificity: + 1. API-level params (api_section['params']) + 2. class-level params (api_section[class_name]['params']) + 3. model-level params (model_data['params']) + 4. agent overrides (agent_params_override) + Returns the merged dict. + """ + api_level_params = api_section.get('params', {}) + class_level_params = api_section[class_name].get('params', {}) + model_level_params = model_data.get('params', {}) - Returns: None + merged_params = {**api_level_params, **class_level_params, **model_level_params, **agent_params_override} + return merged_params - Raises: - FileNotFoundError: If a required configuration or persona file is not found. - KeyError: If a required key is missing in the configuration. - Exception: For general errors encountered during the loading process. - """ - try: - self.reload() - - agent = self.find_agent_config(agent_name) - api, model, final_model_params = self.resolve_model_overrides(agent) - llm = self.get_llm(api, model) - persona_data = self.load_persona(agent) - prompts = self.fix_prompt_placeholders(agent.get('Prompts', {})) - - return { - 'name': agent_name, - 'settings': self.data['settings'], - 'llm': llm, - 'params': final_model_params, - 'persona': persona_data, - 'prompts': prompts, - } - except FileNotFoundError as e: - raise FileNotFoundError(f"Configuration or persona file not found: {e}") - except KeyError as e: - raise KeyError(f"Missing key in configuration: {e}") - except Exception as e: - raise Exception(f"Error loading agent data: {e}") + # ------------------------------------------------------------------------ + # Model Handling + # ------------------------------------------------------------------------ - def reload(self): + @staticmethod + def get_model(api_name: str, class_name: str, model_identifier: str) -> Any: """ - Reloads configurations for an agent. + Dynamically imports and instantiates the Python class for the requested API/class/identifier. + We assume: + - The Python module name is derived from the API’s name (e.g. openai_api). + - The Python class name is exactly the same as the key used under that API (e.g. openai_api -> "O1Series", "GPT", etc.). + - The model’s identifier is a valid identifier. """ - if self.data['settings']['system']['OnTheFly']: - self.load_all_configurations() + # Actually import: from .apis import + module = importlib.import_module(f".apis.{api_name}", package=__package__) + model_class = getattr(module, class_name) - def resolve_model_overrides(self, agent: dict) -> tuple[str, str, dict]: + # Instantiate the model with the identifier + return model_class(model_identifier) + + # ------------------------------------------------------------------------ + # Utility Methods + # ------------------------------------------------------------------------ + + def find_config(self, category: str, config_name: str) -> Optional[Dict[str, Any]]: """ - Resolves the model and API overrides for the agent, if any. + General method to search for a configuration by name within a specified category. Parameters: - agent (dict): The agent's configuration data. + category (str): The category to search in (e.g., 'prompts', 'flows'). + config_name (str): The name of the configuration to find. Returns: - tuple: The resolved API, model, and final model parameters. + dict: The configuration dictionary for the specified name, or None if not found. """ - settings = self.data['settings'] + def search_nested_dict(nested_dict, target): + for key, value in nested_dict.items(): + if key == target: + return value + elif isinstance(value, dict): + result = search_nested_dict(value, target) + if result is not None: + return result + return None - model_overrides = agent.get('ModelOverrides', {}) - model_settings = settings['models']['ModelSettings'] + config = search_nested_dict(self.data.get(category, {}), config_name) + if not config: + raise FileNotFoundError(f"Config '{config_name}' not found in configuration.") - api = model_overrides.get('API', model_settings['API']) - model = model_overrides.get('Model', model_settings['Model']) + return config - default_params = model_settings['Params'] - params = settings['models']['ModelLibrary'].get(api, {}).get('models', {}).get(model, {}).get('params', {}) + @staticmethod + def get_nested_dict(data: dict, path_parts: tuple): + """ + Gets or creates a nested dictionary given the parts of a relative path. - combined_params = {**default_params, **params} - final_model_params = {**combined_params, **model_overrides.get('Params', {})} + Args: + data (dict): The top-level dictionary to start from. + path_parts (tuple): A tuple of path components leading to the desired nested dictionary. - return api, model, final_model_params + Returns: + A reference to the nested dictionary at the end of the path. + """ + for part in path_parts: + if part not in data: + data[part] = {} + data = data[part] + return data def fix_prompt_placeholders(self, prompts): """ @@ -354,24 +403,71 @@ def fix_prompt_placeholders(self, prompts): if re.match(self.pattern, key): # Convert to a string placeholder return f"{{{key}}}" - else: - # If not a valid variable name, process it normally - fixed_prompts = {} - for k, v in prompts.items(): - fixed_key = self.fix_prompt_placeholders(k) - fixed_value = self.fix_prompt_placeholders(v) - fixed_prompts[fixed_key] = fixed_value - return fixed_prompts - else: - fixed_prompts = {} - for key, value in prompts.items(): - fixed_key = self.fix_prompt_placeholders(key) - fixed_value = self.fix_prompt_placeholders(value) - fixed_prompts[fixed_key] = fixed_value - return fixed_prompts - elif isinstance(prompts, list): + + # If not a valid variable name, process it normally + fixed_prompts = {} + for key, value in prompts.items(): + fixed_key = self.fix_prompt_placeholders(key) + fixed_value = self.fix_prompt_placeholders(value) + fixed_prompts[fixed_key] = fixed_value + return fixed_prompts + + if isinstance(prompts, list): return [self.fix_prompt_placeholders(item) for item in prompts] - elif prompts is None: + + if not prompts: return '' - else: - return prompts + + return prompts + + def reload(self): + """ + Reloads configurations if on-the-fly reloading is enabled. + """ + if self.data['settings']['system']['misc'].get('on_the_fly', False): + self.load_all_configurations() + + def find_file_in_directory(self, directory: str, filename: str): + """ + Recursively searches for a file within a directory and its subdirectories. + + Parameters: + directory (str): The directory to search in. + filename (str): The name of the file to find. + + Returns: + pathlib.Path or None: The full path to the file if found, None otherwise. + """ + directory = pathlib.Path(pathlib.Path(self.config_path) / directory) + + for file_path in directory.rglob(filename): + return file_path + return None + + def load_persona(self, agent_config: dict) -> Optional[Dict[str, Any]]: + """ + Loads the persona for the agent, if personas are enabled. + + Parameters: + agent_config (dict): The agent's configuration data. + + Returns: + dict: The loaded persona data. + + Raises: + FileNotFoundError: If the persona file is not found. + """ + settings = self.data['settings'] + if not settings['system']['persona'].get('enabled', False): + return None + + persona_file = agent_config.get('persona') or settings['system']['persona'].get('name', 'default') + if persona_file not in self.data.get('personas', {}): + raise FileNotFoundError( + f"Selected Persona '{persona_file}' not found. " + "Please make sure the corresponding persona file is in the personas folder" + ) + + return self.data['personas'][persona_file] + + diff --git a/src/agentforge/init_agentforge.py b/src/agentforge/init_agentforge.py index 9726bc5a..905a479e 100755 --- a/src/agentforge/init_agentforge.py +++ b/src/agentforge/init_agentforge.py @@ -4,95 +4,126 @@ import importlib.util from pathlib import Path -def copy_directory(root, src, dst, override_all=False, skip_all=False): + +def user_decision_prompt(existing_file: str) -> str: + """ + Interactively prompts the user about what to do with a file conflict. + Returns one of the following single-character codes: + - 'y' for overriding the file + - 'n' for skipping this file + - 'a' for overriding all existing files without asking again + - 'z' for skipping all existing files without asking again + - '' (empty) if the user input is invalid + """ + print(f"\nFile '{existing_file}' already exists and is different from the source.") + response = input( + "Select an option:\n" + "[Y] Override this file\n" + "[N] Skip this file\n" + "[A] Override all existing files without asking again\n" + "[Z] Skip all existing files without asking again\n" + "Enter your choice (Y/N/A/Z): " + ).lower() + valid_choices = {'y', 'n', 'a', 'z'} + if response in valid_choices: + return response + print("Invalid option. Skipping this file by default.") + return '' + + +def should_copy_file( + src_file: str, + dst_file: str, + skip_all: bool, + override_all: bool +) -> (bool, bool, bool): + """ + Determines whether to copy a file from src_file to dst_file based on existing + state flags and user decision. Returns a tuple of three booleans in the form: + (copy_this_file, new_skip_all, new_override_all). """ - Copies a directory from src to dst, including all subdirectories and files, - with options to override or skip existing files. + if skip_all: + # We’re skipping all conflicts globally, no copy, just return the updated flags. + return False, skip_all, override_all + + if not os.path.exists(dst_file): + # If there's no existing file, proceed with the copy. + return True, skip_all, override_all + + # If the files match, skip copying; there's no reason to replace it. + if filecmp.cmp(src_file, dst_file, shallow=False): + return False, skip_all, override_all + + # If we’re overriding all conflicts globally, skip user prompt and copy. + if override_all: + return True, skip_all, override_all + + # Otherwise, prompt the user for a decision. + decision = user_decision_prompt(os.path.relpath(dst_file)) + if decision == 'a': + # Override all from now on. + return True, skip_all, True + if decision == 'z': + # Skip all from now on. + return False, True, override_all + if decision == 'n': + # Skip just this file. + return False, skip_all, override_all + if decision == 'y': + # Copy just this file. + return True, skip_all, override_all + + # If user input is invalid or empty, skip the file by default. + return False, skip_all, override_all + + +def copy_directory( + root: Path, + src: Path, + override_all: bool = False, + skip_all: bool = False +) -> None: """ - for src_dir, dirs, files in os.walk(src): - # Skip __pycache__ directories + Recursively copies files from 'src' to 'root', skipping __pycache__ and __init__.py + or .pyc files, while respecting user choices about overwriting. + """ + for current_dir, dirs, files in os.walk(src): dirs[:] = [d for d in dirs if d != '__pycache__'] - dst_dir = src_dir.replace(str(src), str(dst), 1) + dst_dir = current_dir.replace(str(src), str(root), 1) if not os.path.exists(dst_dir): os.makedirs(dst_dir) - print(f"Created directory '{os.path.relpath(dst_dir, start=dst)}'.") + print(f"Created directory '{os.path.relpath(dst_dir, start=root)}'.") for file_ in files: - # Skip __init__.py files and any .pyc files if file_ == '__init__.py' or file_.endswith('.pyc'): continue - src_file_path = os.path.join(src_dir, file_) - dst_file_path = os.path.join(dst_dir, file_) - - # Ensure the file paths are strings if they're not already - src_file_str = str(src_file_path) - dst_file_str = str(dst_file_path) + src_file_str = str(os.path.join(current_dir, file_)) + dst_file_str = str(os.path.join(dst_dir, file_)) relative_src_path = os.path.relpath(src_file_str, start=src) - relative_dst_path = os.path.relpath(dst_file_str, start=dst) - - if os.path.exists(dst_file_str): - # Compare the files - if filecmp.cmp(src_file_str, dst_file_str, shallow=False): - # Files are the same, skip copying - print(f"No changes detected in '{relative_dst_path}'. Skipping.") - continue - else: - # Files are different - if skip_all: - print(f"Skipped '{relative_dst_path}' (skip all is enabled).") - continue - elif override_all: - # Proceed to copy - pass - else: - # Prompt the user for action - print(f"\nFile '{relative_dst_path}' already exists and is different from the source.") - response = input( - "Select an option:\n" - "[Y] Override this file\n" - "[N] Skip this file\n" - "[A] Override all existing files without asking again\n" - "[Z] Skip all existing files without asking again\n" - "Enter your choice (Y/N/A/Z): " - ).lower() - - if response == 'a': - override_all = True - # Proceed to copy - elif response == 'z': - skip_all = True - print(f"Skipped '{relative_dst_path}' (skip all is enabled).") - continue - elif response == 'n': - print(f"Skipped '{relative_dst_path}'.") - continue - elif response == 'y': - # Proceed to copy - pass - else: - print("Invalid option. Skipping this file.") - continue - else: - # Destination file does not exist, proceed to copy - pass - - if skip_all: - print(f"Skipped '{relative_dst_path}' (skip all is enabled).") - continue + relative_dst_path = os.path.relpath(dst_file_str, start=root) - # Proceed to copy the file - shutil.copy2(src_file_str, dst_file_str) # copy2 is used to preserve metadata + do_copy, skip_all, override_all = should_copy_file( + src_file_str, + dst_file_str, + skip_all, + override_all + ) + if not do_copy: + print(f"Skipped '{relative_dst_path}'.") + continue + + shutil.copy2(src_file_str, dst_file_str) print(f"Copied '{relative_src_path}' to '{relative_dst_path}'.") -def setup_agentforge(): +def setup_agentforge() -> None: """ - Sets up the AgentForge project directory by copying the setup_files directory. + Locates the AgentForge package, copies its 'setup_files' directory into + the current working directory, and provides feedback on the process. """ - # Identify the AgentForge library path package_name = 'agentforge' try: spec = importlib.util.find_spec(package_name) @@ -101,19 +132,12 @@ def setup_agentforge(): return agentforge_path = spec.submodule_search_locations[0] - print(f"Found {package_name} at {agentforge_path}") - - # Define source and destination paths installer_path = os.path.join(agentforge_path, 'setup_files') project_root = Path.cwd() - destination_path = project_root / '.agentforge' - - # Copy the setup_files directory to the project directory - copy_directory(project_root, installer_path, destination_path) + copy_directory(project_root, Path(installer_path)) print("AgentForge setup is complete.") - except Exception as e: print(f"An error occurred: {e}") diff --git a/src/agentforge/llm/LMStudio.py b/src/agentforge/llm/LMStudio.py deleted file mode 100644 index 421ef2c9..00000000 --- a/src/agentforge/llm/LMStudio.py +++ /dev/null @@ -1,68 +0,0 @@ -import requests -import json -from agentforge.utils.Logger import Logger - -class LMStudio: - - def __init__(self, model): - """ - Initializes the CustomAPI class. - """ - self._model = model - self.logger = None - - def generate_text(self, model_prompt, **params): - """ - Sends a request to a custom AI model endpoint to generate a completion based on the provided prompt. - - This function constructs a request with specified parameters and sends it to a custom AI endpoint, which is - expected to generate text based on the input prompt. The endpoint URL is read from an environment variable. - - Parameters: - model_prompt (dict[str]): A dictionary containing the model prompts for generating a completion. - **params: Arbitrary keyword arguments for future extensibility, not used currently. - - Returns: - str or None: The JSON response from the AI model if the request is successful, None otherwise. - - Logs the prompt and the response using a Logger instance. If the `CUSTOM_AI_ENDPOINT` environment variable - is not set or if the request fails, appropriate error messages are logged. - """ - self.logger = Logger(name=params.pop('agent_name', 'NamelessAgent')) - self.logger.log_prompt(model_prompt) - - headers = {'Content-Type': 'application/json'} - data = { - "model": self._model, - "messages": [ - {"role": "system", "content": model_prompt.get('System')}, - {"role": "user", "content": model_prompt.get('User')} - ], - "temperature": params["temperature"], - "max_tokens": params["max_new_tokens"], - "stream": False - } - - url = params.pop('host_url', None) - if not url: - self.logger.log("\n\nError: The CUSTOM_AI_ENDPOINT environment variable is not set", 'critical') - - completion = requests.post(url, headers=headers, data=json.dumps(data)) - response_dict = json.loads(completion.text) - message_content = response_dict["choices"][0]["message"]["content"] - - # self.logger.log_response(response.json()['response']) - self.logger.log_response(message_content) - - if completion.status_code == 200: - return message_content - else: - print(f"Request error: {completion}") - return None - - -# ---------------------------------------------------------------------------------------------------- -# Example usage: -# prompt = "What does the cow say?" -# print(request_completion(prompt)) -# print("Done!") diff --git a/src/agentforge/llm/__init__.py b/src/agentforge/llm/__init__.py deleted file mode 100755 index a7ab1650..00000000 --- a/src/agentforge/llm/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from typing import Protocol - - -class LLM(Protocol): - def generate_text(self, prompt, **params): - pass diff --git a/src/agentforge/llm/anthropic.py b/src/agentforge/llm/anthropic.py deleted file mode 100755 index bf740568..00000000 --- a/src/agentforge/llm/anthropic.py +++ /dev/null @@ -1,79 +0,0 @@ -import os -import time -import anthropic -from agentforge.utils.Logger import Logger - -API_KEY = os.getenv('ANTHROPIC_API_KEY') -client = anthropic.Anthropic(api_key=API_KEY) - - -class Claude: - """ - A class for interacting with Anthropic's Claude models to generate text based on provided prompts. - - Manages API calls to Anthropic, handling errors such as rate limits, and retries failed requests with exponential - backoff. - - Attributes: - num_retries (int): The number of times to retry generating text upon encountering errors. - """ - num_retries = 5 - - def __init__(self, model): - """ - Initializes the Claude class with a specific Claude model identifier. - - Parameters: - model (str): The identifier of the Claude model to use for generating text. - """ - self._model = model - self.logger = None - - def generate_text(self, model_prompt, **params): - """ - Generates text based on the provided prompts and additional parameters for the Claude model. - - Parameters: - model_prompt (dict[str]): A dictionary containing the model prompts for generating a completion. - **params: Arbitrary keyword arguments providing additional options to the model such as - 'max_new_tokens', 'temperature', and 'top_p'. - - Returns: - str or None: The generated text from the model or None if the operation fails after retry attempts. - - This method attempts to generate content with the provided prompts and configuration, retrying up to a - specified number of times with exponential backoff in case of errors. It logs the process and outcomes. - """ - self.logger = Logger(name=params.pop('agent_name', 'NamelessAgent')) - self.logger.log_prompt(model_prompt) - - prompt = [{'role': 'user', 'content': model_prompt.get('User')}] - - # Will retry to get chat if a rate limit or bad gateway error is received from the chat - response = None - for attempt in range(self.num_retries): - backoff = 2 ** (attempt + 2) - try: - response = client.messages.create( - messages=prompt, - system=model_prompt.get('System'), - # stop_sequences=[anthropic.HUMAN_PROMPT], - model=self._model, - max_tokens=params["max_new_tokens"], - temperature=params["temperature"], - top_p=params["top_p"] - ) - self.logger.log_response(str(response.content[0].text)) - break - - except Exception as e: - self.logger.log(f"\n\nError: Retrying in {backoff} seconds...\nError Code: {e}", 'warning') - time.sleep(backoff) - - if response is None: - self.logger.log("\n\nError: Failed to get Anthropic Response", 'critical') - else: - usage = str(response.usage) - self.logger.log(f"Claude Token Usage: {usage}\n", 'debug') - - return response.content[0].text diff --git a/src/agentforge/llm/bakollama.py b/src/agentforge/llm/bakollama.py deleted file mode 100644 index 2ebf96e0..00000000 --- a/src/agentforge/llm/bakollama.py +++ /dev/null @@ -1,58 +0,0 @@ -import requests -from agentforge.utils.Logger import Logger - - -class BakoLlama: - - def __init__(self, model): - """ - Initializes the CustomAPI class. - """ - self._model = model - self.logger = None - - def generate_text(self, model_prompt, **params): - """ - Sends a request to a custom AI model endpoint to generate a completion based on the provided prompt. - - This function constructs a request with specified parameters and sends it to a custom AI endpoint, which is - expected to generate text based on the input prompt. The endpoint URL is read from an environment variable. - - Parameters: - model_prompt (dict[str]): A dictionary containing the model prompts for generating a completion. - **params: Arbitrary keyword arguments for future extensibility, not used currently. - - Returns: - str or None: The JSON response from the AI model if the request is successful, None otherwise. - - Logs the prompt and the response using a Logger instance. If the `CUSTOM_AI_ENDPOINT` environment variable - is not set or if the request fails, appropriate error messages are logged. - """ - self.logger = Logger(name=params.pop('agent_name', 'NamelessAgent')) - self.logger.log_prompt(model_prompt) - - headers = {'Content-Type': 'application/json'} - data = { - "temperature": params["temperature"], - "model": self._model, - "messages": [ - {"role": "system", "content": model_prompt.get('System')}, - {"role": "user", "content": model_prompt.get('User')} - ], - "max_tokens": params["max_new_tokens"], - "stream": False - } - - url = params.pop('host_url', None) - if not url: - self.logger.log(f"\n\nError: The CUSTOM_AI_ENDPOINT environment variable is not set: {url}", 'critical') - - response = requests.post(url, headers=headers, json=data) - result = response.json()['choices'][0]['message']['content'] - self.logger.log_response(result) - - if response.status_code == 200: - return result - else: - print(f"Request error: {response}") - return None \ No newline at end of file diff --git a/src/agentforge/llm/claude_old.py b/src/agentforge/llm/claude_old.py deleted file mode 100755 index f20e46ab..00000000 --- a/src/agentforge/llm/claude_old.py +++ /dev/null @@ -1,92 +0,0 @@ -import os -import time -import anthropic -from agentforge.utils.Logger import Logger - -API_KEY = os.getenv('ANTHROPIC_API_KEY') -client = anthropic.Anthropic(api_key=API_KEY) - - -def parse_prompts(prompts): - """ - Formats a list of prompts into a single string formatted specifically for Anthropic's AI models. - - Parameters: - prompts (dict[str]): A dictionary containing the model prompts for generating a completion. - - Returns: - str: A formatted prompt string combining human and AI prompt indicators with the original prompt content. - """ - prompt = ''.join(prompts) - prompt = f"{anthropic.HUMAN_PROMPT} {prompt}{anthropic.AI_PROMPT}" - - return prompt - - -class Claude: - """ - A class for interacting with Anthropic's Claude models to generate text based on provided prompts. - - Manages API calls to Anthropic, handling errors such as rate limits, and retries failed requests with exponential - backoff. - - Attributes: - num_retries (int): The number of times to retry generating text upon encountering errors. - """ - num_retries = 5 - - def __init__(self, model): - """ - Initializes the Claude class with a specific Claude model identifier. - - Parameters: - model (str): The identifier of the Claude model to use for generating text. - """ - self._model = model - self.logger = None - - def generate_text(self, model_prompt, **params): - """ - Generates text based on the provided prompts and additional parameters for the Claude model. - - Parameters: - model_prompt (dict[str]): A dictionary containing the model prompts for generating a completion. - **params: Arbitrary keyword arguments providing additional options to the model such as - 'max_new_tokens', 'temperature', and 'top_p'. - - Returns: - str or None: The generated text from the model or None if the operation fails after retry attempts. - - This method attempts to generate content with the provided prompts and configuration, retrying up to a - specified number of times with exponential backoff in case of errors. It logs the process and outcomes. - """ - self.logger = Logger(name=params.pop('agent_name', 'NamelessAgent')) - self.logger.log_prompt(model_prompt) - - prompt = parse_prompts(model_prompt) - - # Will retry to get chat if a rate limit or bad gateway error is received from the chat - response = None - for attempt in range(self.num_retries): - backoff = 2 ** (attempt + 2) - try: - response = client.completions.create( - prompt=prompt, - # stop_sequences=[anthropic.HUMAN_PROMPT], - model=self._model, - max_tokens_to_sample=params["max_new_tokens"], - temperature=params["temperature"], - top_p=params["top_p"] - ) - # print(f"Response:{response}\n") - self.logger.log_response(response.completion) - break - - except Exception as e: - self.logger.log(f"\n\nError: Retrying in {backoff} seconds...\nError Code: {e}", 'warning') - time.sleep(backoff) - - if response is None: - self.logger.log("\n\nError: Failed to get Anthropic Response", 'critical') - - return response.completion diff --git a/src/agentforge/llm/gemini.py b/src/agentforge/llm/gemini.py deleted file mode 100755 index 497b6b7c..00000000 --- a/src/agentforge/llm/gemini.py +++ /dev/null @@ -1,87 +0,0 @@ -import os -import time -import google.generativeai as genai -from google.generativeai.types import HarmCategory, HarmBlockThreshold -from agentforge.utils.Logger import Logger - -# Get API key from Env -GOOGLE_API_KEY = os.getenv('GOOGLE_API_KEY') -genai.configure(api_key=GOOGLE_API_KEY) - - -class Gemini: - """ - A class for interacting with Google's Generative AI models to generate text based on provided prompts. - - Handles API calls to Google's Generative AI, including error handling for rate limits and retries failed requests. - - Attributes: - num_retries (int): The number of times to retry generating text upon encountering errors. - """ - num_retries = 4 - - def __init__(self, model): - """ - Initializes the Gemini class with a specific Generative AI model from Google. - - Parameters: - model (str): The identifier of the Google Generative AI model to use for generating text. - """ - self._model = genai.GenerativeModel(model) - self.logger = None - - def generate_text(self, model_prompt, **params): - """ - Generates text based on the provided prompts and additional parameters for the model. - - Parameters: - model_prompt (dict[str]): A dictionary containing the model prompts for generating a completion. - **params: Arbitrary keyword arguments providing additional options to the model. - - Returns: - str or None: The generated text from the model or None if the operation fails after retry attempts. - - This method attempts to generate content with the provided prompts and configuration, retrying up to a - specified number of times with exponential backoff in case of errors. It logs the process and errors. - """ - self.logger = Logger(name=params.pop('agent_name', 'NamelessAgent')) - self.logger.log_prompt(model_prompt) - - prompt = '\n\n'.join(model_prompt) - - # Will retry to get chat if a rate limit or bad gateway error is received from the chat - reply = None - for attempt in range(self.num_retries): - backoff = 8 ** (attempt + 2) - try: - response = self._model.generate_content( - prompt, - safety_settings={ - HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE, - HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE, - HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE, - HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE, - }, - generation_config=genai.types.GenerationConfig( - max_output_tokens=params["max_new_tokens"], - temperature=params["temperature"], - top_p=params.get("top_p", 1), - top_k=params.get("top_k", 1), - candidate_count=max(params.get("candidate_count", 1),1) - ) - ) - - reply = response.text - self.logger.log_response(reply) - - break - - except Exception as e: - self.logger.log(f"\n\nError: Retrying in {backoff} seconds...\nError Code: {e}", 'warning') - time.sleep(backoff) - - # reply will be none if we have failed above - if reply is None: - self.logger.log("\n\nError: Failed to get Gemini Response", 'critical') - - return reply diff --git a/src/agentforge/llm/groq_api.py b/src/agentforge/llm/groq_api.py deleted file mode 100644 index 8f97702e..00000000 --- a/src/agentforge/llm/groq_api.py +++ /dev/null @@ -1,62 +0,0 @@ -import os -from groq import Groq -from agentforge.utils.Logger import Logger - - -class GroqAPI: - - def __init__(self, model): - """ - Initializes the CustomAPI class. - """ - self._model = model - self.prompt_log = None - self.prompt = None - self.logger = None - self.logger2 = None - - def generate_text(self, model_prompt, **params): - """ - Sends a request to a custom AI model endpoint to generate a completion based on the provided prompt. - - This function constructs a request with specified parameters and sends it to a custom AI endpoint, which is - expected to generate text based on the input prompt. The endpoint URL is read from an environment variable. - - Parameters: - model_prompt (dict[str]): A dictionary containing the model prompts for generating a completion. - **params: Arbitrary keyword arguments for future extensibility, not used currently. - - Returns: - str or None: The JSON response from the AI model if the request is successful, None otherwise. - - Logs the prompt and the response using a Logger instance. If the `CUSTOM_AI_ENDPOINT` environment variable - is not set or if the request fails, appropriate error messages are logged. - """ - self.logger = Logger(name=params.pop('agent_name', 'NamelessAgent')) - self.logger.log_prompt(model_prompt) - - api_key = os.getenv("GROQ_API_KEY") - # url = "https://api.groq.com/openai/v1/models" - client = Groq(api_key=api_key) - - response = client.chat.completions.create( - messages=[ - {"role": "system", "content": model_prompt.get('System')}, - {"role": "user", "content": model_prompt.get('User')} - ], - model=self._model, - max_tokens=params['max_new_tokens'], - seed=params['seed'], - stop=params['stop'], - temperature=params['temperature'], - top_p=params['top_p'], - ) - - response_text = response.choices[0].message.content - self.logger.log_response(response_text) - - if response.choices and response.choices[0].message: - return response_text - else: - self.logger.log(f"Request error: {response}", 'error') - return response \ No newline at end of file diff --git a/src/agentforge/llm/ollama.py b/src/agentforge/llm/ollama.py deleted file mode 100755 index db5d271b..00000000 --- a/src/agentforge/llm/ollama.py +++ /dev/null @@ -1,63 +0,0 @@ -import requests -from agentforge.utils.Logger import Logger - - -class Ollama: - - def __init__(self, model): - """ - Initializes the CustomAPI class. - """ - self._model = model - self.logger = None - - def generate_text(self, model_prompt, **params): - """ - Sends a request to a custom AI model endpoint to generate a completion based on the provided prompt. - - This function constructs a request with specified parameters and sends it to a custom AI endpoint, which is - expected to generate text based on the input prompt. The endpoint URL is read from an environment variable. - - Parameters: - model_prompt (dict[str]): A dictionary containing the model prompts for generating a completion. - **params: Arbitrary keyword arguments for future extensibility, not used currently. - - Returns: - str or None: The JSON response from the AI model if the request is successful, None otherwise. - - Logs the prompt and the response using a Logger instance. If the `CUSTOM_AI_ENDPOINT` environment variable - is not set or if the request fails, appropriate error messages are logged. - """ - self.logger = Logger(name=params.pop('agent_name', 'NamelessAgent')) - self.logger.log_prompt(model_prompt) - - headers = {'Content-Type': 'application/json'} - data = { - "temperature": params["temperature"], - "model": self._model, - "system": model_prompt.get('System'), - "prompt": model_prompt.get('User'), - "max_tokens": params["max_new_tokens"], - "stream": False - } - - url = params.pop('host_url', None) - if not url: - self.logger.log(f"\n\nError: The CUSTOM_AI_ENDPOINT environment variable is not set: {url}", 'critical') - - response = requests.post(url, headers=headers, json=data) - result = response.json()['choices'][0]['message']['content'] - self.logger.log_response(result) - - if response.status_code == 200: - return result - else: - print(f"Request error: {response}") - return None - - -# ---------------------------------------------------------------------------------------------------- -# Example usage: -# prompt = "What does the cow say?" -# print(request_completion(prompt)) -# print("Done!") diff --git a/src/agentforge/llm/oobabooga.py b/src/agentforge/llm/oobabooga.py deleted file mode 100755 index a2512997..00000000 --- a/src/agentforge/llm/oobabooga.py +++ /dev/null @@ -1,71 +0,0 @@ -# import sseclient # pip install sseclient-py -import requests -from agentforge.utils.Logger import Logger - - -class Oobabooga: - """ - A class for interacting with an external text generation service named Oobabooga. This class handles the - construction and execution of requests to the Oobabooga API to generate text based on a given prompt. - - Attributes: - _model (str): The model identifier used for generating text. This is kept for compatibility but may not - be directly used depending on the external service's API. - """ - def __init__(self, model): - """ - Initializes the Oobabooga class with a specific model identifier. - - Parameters: - model (str): The identifier of the model to use for text generation. - """ - self._model = model - self.logger = None - - def generate_text(self, model_prompt, **params): - """ - Generates text based on the provided prompt and additional parameters by making a request to the - Oobabooga service's API. - - Parameters: - model_prompt (dict[str]): A dictionary containing the model prompts for generating a completion. - **params: Arbitrary keyword arguments providing additional options for the request. This includes - 'agent_name' for logging purposes and 'host_url' for specifying the Oobabooga service's address. - - Returns: - str: The generated text from the Oobabooga service. - - Raises: - Exception: Logs a critical error message if an exception occurs during the API request. - """ - self.logger = Logger(name=params.pop('agent_name', 'NamelessAgent')) - self.logger.log_prompt(model_prompt) - - prompt = '\n\n'.join(model_prompt) - - # Server address - host = params.pop('host_url', None) - url = f"{host}/v1/chat/completions" - - headers = {"Content-Type": "application/json"} - - message = [{"role": "user", "content": prompt}] - - data = { - "mode": "chat", - "character": "Example", - "messages": message - } - - reply = '' - try: - response = requests.post(url, headers=headers, json=data, verify=False) - - reply = response.json()['choices'][0]['message']['content'] - self.logger.log_response(reply) - - - except Exception as e: - self.logger.log(f"\n\nError: {e}", 'critical') - - return reply diff --git a/src/agentforge/llm/openai.py b/src/agentforge/llm/openai.py deleted file mode 100755 index 405462cb..00000000 --- a/src/agentforge/llm/openai.py +++ /dev/null @@ -1,93 +0,0 @@ -import time -from openai import OpenAI, APIError, RateLimitError, APIConnectionError -from agentforge.utils.Logger import Logger - -# Assuming you have set OPENAI_API_KEY in your environment variables -client = OpenAI() - - -class GPT: - """ - A class for interacting with OpenAI's GPT models to generate text based on provided prompts. - - Handles API calls to OpenAI, including error handling for rate limits and API connection issues, and retries - failed requests. - - Attributes: - num_retries (int): The number of times to retry generating text upon encountering rate limits or - connection errors. - """ - num_retries = 5 - - def __init__(self, model): - """ - Initializes the GPT class with a specific model. - - Parameters: - model (str): The identifier of the GPT model to use for generating text. - """ - self._model = model - self.logger = None - - def generate_text(self, model_prompt, **params): - """ - Generates text based on the provided prompts and additional parameters for the GPT model. - - Parameters: - model_prompt (dict[str]): A dictionary containing the model prompts for generating a completion. - **params: Arbitrary keyword arguments providing additional options to the model (e.g., temperature, max tokens). - - Returns: - str or None: The generated text from the GPT model or None if the operation fails. - - Raises: - APIError: If an API error occurs not related to rate limits or bad gateway responses. - """ - self.logger = Logger(name=params.pop('agent_name', 'NamelessAgent')) - self.logger.log_prompt(model_prompt) - - prompt = [ - {"role": "system", "content": model_prompt.get('System')}, - {"role": "user", "content": model_prompt.get('User')} - ] - - # Will retry to get chat if a rate limit or bad gateway error is received from the chat - reply = None - for attempt in range(self.num_retries): - backoff = 2 ** (attempt + 2) - try: - response = client.chat.completions.create( - model=self._model, - messages=prompt, - max_tokens=params["max_new_tokens"], - n=params["n"], - temperature=params["temperature"], - top_p=params["top_p"], - presence_penalty=params["penalty_alpha"], - stop=params["stop"], - ) - - reply = response.choices[0].message.content - self.logger.log_response(reply) - - break - - except RateLimitError: - self.logger.log("\n\nError: Reached API rate limit, retrying in 20 seconds...", 'warning') - time.sleep(20) - except APIConnectionError: - self.logger.log("\n\nError: Connection issue, retrying in 2 seconds...", 'warning') - time.sleep(2) - except APIError as e: - if getattr(e, 'status_code', None) == 502: - self.logger.log("\n\nError: Connection issue, retrying in 2 seconds...", 'warning') - time.sleep(backoff) - else: - raise - - # reply will be none if we have failed above - if reply is None: - self.logger.log("\n\nError: Failed to get OpenAI Response", 'critical') - - return reply - diff --git a/src/agentforge/llm/openrouter.py b/src/agentforge/llm/openrouter.py deleted file mode 100644 index ce9b8630..00000000 --- a/src/agentforge/llm/openrouter.py +++ /dev/null @@ -1,100 +0,0 @@ -import os -import time -import requests -from agentforge.utils.Logger import Logger - -# Get the API key from the environment variable -api_key = os.getenv('OPENROUTER_API_KEY') - - -class OpenRouter: - """ - A class for interacting with OpenRouter's API to generate text based on provided prompts. - - Handles API calls to OpenRouter, including error handling and retries for failed requests. - - Attributes: - num_retries (int): The number of times to retry generating text upon encountering errors. - """ - num_retries = 5 - - def __init__(self, model): - """ - Initializes the OpenRouter class with a specific model. - - Parameters: - model (str): The identifier of the model to use for generating text. - """ - self._model = model - self.logger = None - - def generate_text(self, model_prompt, **params): - """ - Generates text based on the provided prompts and additional parameters for the OpenRouter model. - - Parameters: - model_prompt (dict[str]): A dictionary containing the model prompts for generating a completion. - **params: Arbitrary keyword arguments providing additional options to the model and API call. - - Returns: - str or None: The generated text from the OpenRouter model or None if the operation fails. - """ - self.logger = Logger(name=params.pop('agent_name', 'NamelessAgent')) - self.logger.log_prompt(model_prompt) - - prompt = [ - {"role": "system", "content": model_prompt.get('System')}, - {"role": "user", "content": model_prompt.get('User')} - ] - - headers = { - "Authorization": f"Bearer {api_key}", - "Content-Type": "application/json", - "HTTP-Referer": params.get("http_referer", ""), - "X-Title": 'AgentForge' - } - - data = { - "model": self._model, - "messages": prompt, - "max_tokens": params.get("max_new_tokens"), - "temperature": params.get("temperature"), - "top_p": params.get("top_p"), - "presence_penalty": params.get("penalty_alpha"), - "stop": params.get("stop"), - "stream": params.get("stream", False) - } - - reply = None - for attempt in range(self.num_retries): - backoff = 2 ** (attempt + 2) - try: - response = requests.post( - "https://openrouter.ai/api/v1/chat/completions", - headers=headers, - json=data - ) - response.raise_for_status() - reply = response.json()['choices'][0]['message']['content'] - self.logger.log_response(reply) - break - - except requests.exceptions.RequestException as e: - error_message = f"\n\nError: {str(e)}" - try: - error_message += f"\nResponse JSON: {response.json()}" - except ValueError: - error_message += "\nUnable to parse response JSON" - - if response.status_code == 429: # Rate limit error - self.logger.log(error_message + f", retrying in {backoff} seconds...", 'warning') - elif response.status_code == 502: # Bad gateway - self.logger.log(error_message + f", retrying in {backoff} seconds...", 'warning') - else: - self.logger.log(error_message + f", retrying in {backoff} seconds...", 'warning') - time.sleep(backoff) - - if reply is None: - self.logger.log("\n\nError: Failed to get OpenRouter Response", 'critical') - - return reply diff --git a/src/agentforge/modules/InjectKG.py b/src/agentforge/modules/InjectKG.py deleted file mode 100755 index 17dc8fa2..00000000 --- a/src/agentforge/modules/InjectKG.py +++ /dev/null @@ -1,69 +0,0 @@ -from agentforge.agents.MetadataKGAgent import MetadataKGAgent -from agentforge.utils.storage_interface import StorageInterface -import uuid -from typing import Optional, Any, Dict - - -class Consume: - - def __init__(self): - self.metadata_extract = MetadataKGAgent() - self.storage = StorageInterface().storage_utils - - def consume(self, knowledge_base_name: str, sentence: str, reason: str, source_name: str, source_path: str, - chunk: Optional[Any] = None, existing_knowledge: Optional[str] = None) -> Dict[str, Any]: - """ - Process and store a given sentence along with its metadata in a specified knowledge base. - - This method extracts triples and other relevant metadata from the given sentence, - generates a unique identifier, builds a parameter dictionary with the sentence and its metadata, - and then saves this information using the storage's save_memory method. - Finally, it prints and returns the constructed parameters. - - Parameters: - knowledge_base_name (str): The name of the knowledge base collection to store the data. - sentence (str): The sentence to be processed and stored. - reason (str): The reason or context for storing this sentence. - source_name (str): The name of the source from which the sentence is derived. - source_path (str): The path of the source from which the sentence is derived. - chunk (Optional[Any]): Additional optional chunk data to be used in triple extraction, defaults to None. - existing_knowledge (Optional[Any]): Additional existing knowledge to be used in triple extraction, - defaults to None. - - Returns: - Dict[str, Any]: A dictionary containing the input data along with the generated metadata. - """ - - # Extract Metadata - nodes = self.metadata_extract.run(sentence=sentence, text_chunk=chunk, existing_knowledge=existing_knowledge) - - # build params - random_uuid = str(uuid.uuid4()) - params = { - "collection_name": knowledge_base_name, - "data": [sentence], - "ids": [f"{random_uuid}"], - "metadata": [{ - "id": f"{random_uuid}", - "reason": reason, - "sentence": sentence, - "source_name": source_name, - "source_path": source_path, - **nodes - }] - } - - # Output preparation and printing - output = params.copy() - metadata_values = output["metadata"][0] # Access the first (and only) metadata item - if isinstance(metadata_values, dict): - for key, value in metadata_values.items(): - print(f"{key}: {value}") - else: - print("Error: metadata_values is not a dictionary") - - # Saving the data - self.storage.save_memory(**params) - - return output - diff --git a/src/agentforge/modules/KnowledgeTraversal.py b/src/agentforge/modules/KnowledgeTraversal.py deleted file mode 100755 index c362c2e8..00000000 --- a/src/agentforge/modules/KnowledgeTraversal.py +++ /dev/null @@ -1,130 +0,0 @@ -from agentforge.utils.Logger import Logger -from typing import Any, Dict -from ..utils.ChromaUtils import ChromaUtils - -def merge_dictionaries_by_appending_unique_entries(target_dict: dict, source_dict: dict) -> dict: - """ - Merges two dictionaries by appending non-duplicate entries from the source dictionary to the target dictionary. - - This function is specifically designed to merge entries based on unique ID values. It appends data from the source - dictionary to the target dictionary only for those IDs that are unique to the source dictionary, ensuring there is - no duplication of IDs and their corresponding data is properly concatenated. - - Parameters: - - target_dict (dict): The dictionary into which data will be merged. It should follow a specific structure: - a list containing a single list of IDs under the 'ids' key, and similar list-based structures under other - keys. - - source_dict (dict): The dictionary from which data will be extracted and merged into the target dictionary. - It should follow the same structural format as the target dictionary. - - Returns: - - dict: The updated target dictionary, now including the original data plus any unique, non-duplicated data - from the source dictionary. - - The function iterates over the IDs and corresponding data in the source dictionary. If the data under any key is - not None it appends the data associated with those IDs from the source dictionary to the respective keys in the - target dictionary. - - Notes: - - The function assumes that each 'ids' list contains unique identifiers. - - It is presumed that the data structure under each key across both dictionaries is consistent, typically - lists containing a single sublist of any data type. - - The function also assumes that the order of IDs and their corresponding data points is maintained across - all keys in both dictionaries. - """ - - target_ids_set = set(target_dict['ids'][0]) - new_ids = [id_ for id_ in source_dict['ids'][0] if id_ not in target_ids_set] - - for key, values in source_dict.items(): - if key not in target_dict: - target_dict[key] = [[]] - - if values: - for id_ in new_ids: - index = source_dict['ids'][0].index(id_) - target_dict[key][0].extend(values[0][index:index + 1]) - - return target_dict - - -class KnowledgeTraversal: - - def __init__(self): - """ - Initializes the KnowledgeTraversal class. - - This constructor sets up the KnowledgeTraversal instance with essential components for querying - and logging within a knowledge base environment. It establishes a logger specific to the class - for tracking events and interactions and initializes a storage interface for accessing the - knowledge base. - """ - self.logger = Logger(name=self.__class__.__name__) - self.storage = ChromaUtils() - - def query_knowledge(self, knowledge_base_name: str, query: str, metadata_map: Dict[str, str], - initial_num_results: int = 1, subquery_num_results: int = 1) -> Dict[str, Any]: - """ - Queries the knowledge base using a specified query and metadata mappings to retrieve and - aggregate results. - - This method performs an initial query to the specified knowledge base and then conducts - further sub-queries based on the initial results. It merges all obtained results while - avoiding duplicates, primarily focusing on unique metadata entries. - - Parameters: - knowledge_base_name (str): The name of the knowledge base collection to query. - query (str): The query string to be executed against the knowledge base. - metadata_map (Dict[str, str]): A mapping of metadata field names that dictate how the - results should be filtered and merged. - initial_num_results (int): The number of results to retrieve from the initial query. - subquery_num_results (int): The number of results to retrieve for each sub-query based - on the metadata. - - Returns: - Dict[str, Any]: An aggregated set of query results, which includes unique entries from - both the initial query and all performed sub-queries. - - Raises: - KeyError: If 'metadatas' key is not found in the initial query results, indicating an - issue with the expected result structure. - """ - initial_results = self.storage.query_memory(collection_name=knowledge_base_name, - query=query, - num_results=initial_num_results) - - self.logger.log(f"Initial Results:\n{initial_results}", 'debug') - - try: - metadatas = initial_results['metadatas'][0] - except KeyError: - self.logger.log(f"Metadata not found in the results:\n{initial_results}", 'error') - return {} - - final_results = initial_results.copy() - for metadata in metadatas: - where_map = [{haystack: metadata.get(needle)} for haystack, needle in metadata_map.items()] - - if len(where_map) > 1: - where_map = {"$and": where_map} - else: - where_map = where_map[0] - - self.logger.log(f"Collection name: {knowledge_base_name}\n" - f"Query: {query}\n" - f"Filter: {where_map}\n", - 'debug') - - sub_results = self.storage.query_memory(collection_name=knowledge_base_name, - query=query, - filter_condition=where_map, - num_results=subquery_num_results) - - self.logger.log(f"Sub Results:\n{sub_results}", 'debug') - - final_results = merge_dictionaries_by_appending_unique_entries(final_results, sub_results) - - return final_results - - - diff --git a/src/agentforge/modules/Actions.py b/src/agentforge/modules/actions.py similarity index 96% rename from src/agentforge/modules/Actions.py rename to src/agentforge/modules/actions.py index 3a6bd449..9006cfdd 100644 --- a/src/agentforge/modules/Actions.py +++ b/src/agentforge/modules/actions.py @@ -1,13 +1,11 @@ import traceback from typing import List, Dict, Optional, Union -from agentforge.utils.Logger import Logger -from agentforge.utils.ChromaUtils import ChromaUtils -from agentforge.utils.ParsingUtils import ParsingUtils -from agentforge.agents.ActionSelectionAgent import ActionSelectionAgent -from agentforge.agents.ActionCreationAgent import ActionCreationAgent -from agentforge.agents.ToolPrimingAgent import ToolPrimingAgent +from agentforge.agent import Agent +from agentforge.utils.logger import Logger +from agentforge.storage.chroma_storage import ChromaStorage +from agentforge.utils.parsing_processor import ParsingProcessor from ..config import Config -from ..utils.ToolUtils import ToolUtils +from ..utils.tool_utils import ToolUtils def id_generator(data: List[Dict]) -> List[str]: """ @@ -45,14 +43,14 @@ def __init__(self): # Initialize the logger, storage, and functions self.logger = Logger(name=self.__class__.__name__) self.config = Config() - self.storage = ChromaUtils() + self.storage = ChromaStorage() self.tool_utils = ToolUtils() - self.parsing_utils = ParsingUtils() + self.parsing_utils = ParsingProcessor() # Initialize the agents - self.action_creation = ActionCreationAgent() - self.action_selection = ActionSelectionAgent() - self.priming_agent = ToolPrimingAgent() + self.action_creation = Agent("ActionCreationAgent") + self.action_selection = Agent("ActionSelectionAgent") + self.priming_agent = Agent("ToolPrimingAgent") # Load the actions and tools from the config self.actions = self.initialize_collection('Actions') @@ -253,7 +251,7 @@ def prime_tool_for_action(self, objective: str, action: Union[str, Dict], tool: try: # Load the paths into a dictionary - paths_dict = self.storage.config.data['settings']['system']['Paths'] + paths_dict = self.storage.config.data['settings']['system']['paths'] # Construct the work_paths string by iterating over the dictionary work_paths = None @@ -320,7 +318,7 @@ def run_tools_in_sequence(self, objective: str, action: Dict, if isinstance(payload, Dict) and 'error' in payload: return payload # Stop execution and return the error message - tool_context = payload.get('next_tool_context') + tool_context = payload['thoughts'].get('next_tool_context') results = self.tool_utils.dynamic_tool(tool, payload) # Check if an error occurred diff --git a/src/agentforge/agents/__init__.py b/src/agentforge/setup_files/.agentforge/__init__.py similarity index 100% rename from src/agentforge/agents/__init__.py rename to src/agentforge/setup_files/.agentforge/__init__.py diff --git a/src/agentforge/setup_files/actions/__init__.py b/src/agentforge/setup_files/.agentforge/actions/__init__.py similarity index 100% rename from src/agentforge/setup_files/actions/__init__.py rename to src/agentforge/setup_files/.agentforge/actions/__init__.py diff --git a/Sandbox/.agentforge/actions/WebSearch.yaml b/src/agentforge/setup_files/.agentforge/actions/web_search.yaml similarity index 100% rename from Sandbox/.agentforge/actions/WebSearch.yaml rename to src/agentforge/setup_files/.agentforge/actions/web_search.yaml diff --git a/src/agentforge/setup_files/actions/WriteFile.yaml b/src/agentforge/setup_files/.agentforge/actions/write_file.yaml similarity index 100% rename from src/agentforge/setup_files/actions/WriteFile.yaml rename to src/agentforge/setup_files/.agentforge/actions/write_file.yaml diff --git a/src/agentforge/setup_files/personas/__init__.py b/src/agentforge/setup_files/.agentforge/personas/__init__.py similarity index 100% rename from src/agentforge/setup_files/personas/__init__.py rename to src/agentforge/setup_files/.agentforge/personas/__init__.py diff --git a/src/agentforge/setup_files/personas/default.yaml b/src/agentforge/setup_files/.agentforge/personas/default.yaml similarity index 100% rename from src/agentforge/setup_files/personas/default.yaml rename to src/agentforge/setup_files/.agentforge/personas/default.yaml diff --git a/src/agentforge/setup_files/prompts/__init__.py b/src/agentforge/setup_files/.agentforge/prompts/__init__.py similarity index 100% rename from src/agentforge/setup_files/prompts/__init__.py rename to src/agentforge/setup_files/.agentforge/prompts/__init__.py diff --git a/Sandbox/.agentforge/prompts/modules/ActionCreationAgent.yaml b/src/agentforge/setup_files/.agentforge/prompts/modules/ActionCreationAgent.yaml similarity index 97% rename from Sandbox/.agentforge/prompts/modules/ActionCreationAgent.yaml rename to src/agentforge/setup_files/.agentforge/prompts/modules/ActionCreationAgent.yaml index 43516d35..8709f70c 100644 --- a/Sandbox/.agentforge/prompts/modules/ActionCreationAgent.yaml +++ b/src/agentforge/setup_files/.agentforge/prompts/modules/ActionCreationAgent.yaml @@ -1,18 +1,18 @@ -Prompts: - System: - Description: |- +prompts: + system: + description: |- Your task is to create an action YAML file that will help achieve the following objective using the tools available in your environment. Ensure that each step is clear and logically follows from the previous step to achieve the objective effectively. - Objective: |+ + objective: |+ Objective: {objective} - User: - Context: |+ + user: + context: |+ Use the following context to enhance your understanding of the objective. This context provides necessary background and details to inform the creation of the action YAML file: {context} - Tools: |+ + tools: |+ You have access to the following tools, each with specific commands and arguments. Use these tools to create actions that accomplish the given objective. Below is a detailed description of how to interpret and use the tools: - **Name**: The name of the tool. @@ -22,7 +22,7 @@ Prompts: Here are the tools available for creating actions: {tool_list} - Actions: |+ + actions: |+ **Action Creation Overview:** Actions are sequences of one or more tools that are executed in a specific order to perform complex tasks. They combine the functionality of individual tools into cohesive workflows. Actions are defined in YAML files, facilitating organized development and straightforward execution. @@ -46,7 +46,7 @@ Prompts: **How Actions Work:** Actions are executed by running each tool listed in the `Tools` attribute in sequence. The output from one tool is used as input for the next tool, creating a seamless chain of operations that automates complex procedures efficiently. - Instruction: |+ + instruction: |+ **Instructions:** Review the objective and context if provided, and create a suitable action YAML file that outlines a specific action to achieve the given objective. The action should leverage the tools listed above and be structured to be executed by another process using scripts and API tools. @@ -60,7 +60,7 @@ Prompts: The response should be formatted strictly as a YAML file, following the structure provided below. Do not include any code blocks within the YAML format. - Response: |+ + response: |+ **Response Format:** ```yaml Name: diff --git a/Sandbox/.agentforge/prompts/modules/ActionSelectionAgent.yaml b/src/agentforge/setup_files/.agentforge/prompts/modules/ActionSelectionAgent.yaml similarity index 76% rename from Sandbox/.agentforge/prompts/modules/ActionSelectionAgent.yaml rename to src/agentforge/setup_files/.agentforge/prompts/modules/ActionSelectionAgent.yaml index 2eb084d9..ff6a75ab 100644 --- a/Sandbox/.agentforge/prompts/modules/ActionSelectionAgent.yaml +++ b/src/agentforge/setup_files/.agentforge/prompts/modules/ActionSelectionAgent.yaml @@ -1,17 +1,17 @@ -Prompts: - System: - Task: Your task is to decide whether the following objective requires the use of an action. +prompts: + system: + task: Your task is to decide whether the following objective requires the use of an action. - Objective: |+ + objective: |+ Objective: {objective} - User: - Actions: |+ + user: + actions: |+ Consider the following actions available, including the option to choose "Nothing" if no action is required: {action_list} - Instruction: |+ + instruction: |+ Review the actions in light of the main objective provided. You must recommend the most effective action from the list, or "Nothing" if no action is necessary. @@ -20,11 +20,14 @@ Prompts: Strictly adhere to the response format below. Only provide the selected action, reasoning, and feedback without any additional commentary outside of the allowed fields in the format. - Response: |- + response: |- RESPONSE FORMAT: ```yaml action: reasoning: ``` -Persona: default \ No newline at end of file +# Setting Overrides example +persona: + enabled: true + name: default diff --git a/src/agentforge/setup_files/prompts/modules/ToolPrimingAgent.yaml b/src/agentforge/setup_files/.agentforge/prompts/modules/ToolPrimingAgent.yaml similarity index 92% rename from src/agentforge/setup_files/prompts/modules/ToolPrimingAgent.yaml rename to src/agentforge/setup_files/.agentforge/prompts/modules/ToolPrimingAgent.yaml index 6d31d7be..fb4c3137 100644 --- a/src/agentforge/setup_files/prompts/modules/ToolPrimingAgent.yaml +++ b/src/agentforge/setup_files/.agentforge/prompts/modules/ToolPrimingAgent.yaml @@ -1,34 +1,34 @@ -Prompts: - System: - Task: |- +prompts: + system: + task: |- You are a tool priming agent tasked with preparing a tool for an objective: - Objective: |+ + objective: |+ Objective: {objective} - Action: |- + action: |- To achieve this objective, the following action has been selected: {action} - User: - Tool: |+ + user: + tool: |+ Your task is to prime the '{tool_name}' tool in the context of the selected action. Instructions explaining how to use the tool are as follows: {tool_info} - Path: |+ + path: |+ Your working directories are: {path} - Results: |+ + results: |+ Use the following data from the previous tool result in order to prime the '{tool_name}' tool: {previous_results} - Context: |+ + context: |+ Consider the following context and the sequence of tools within the action: {tool_context} - Instruction: |+ + instruction: |+ Review the sequence of tools and understand how each tool feeds into the next to accomplish the overall objective. You must prime the '{tool_name}' tool using the data from the previous tool results if any and the provided context if given. Prime the tool to prepare it for execution, ensuring that it correctly receives and processes the input from the previous tool in the sequence. Do not attempt to achieve the objective directly; focus on priming the tool as a step toward the overarching goal. diff --git a/src/agentforge/setup_files/prompts/modules/__init__.py b/src/agentforge/setup_files/.agentforge/prompts/modules/__init__.py similarity index 100% rename from src/agentforge/setup_files/prompts/modules/__init__.py rename to src/agentforge/setup_files/.agentforge/prompts/modules/__init__.py diff --git a/src/agentforge/setup_files/settings/__init__.py b/src/agentforge/setup_files/.agentforge/settings/__init__.py similarity index 100% rename from src/agentforge/setup_files/settings/__init__.py rename to src/agentforge/setup_files/.agentforge/settings/__init__.py diff --git a/src/agentforge/setup_files/.agentforge/settings/models.yaml b/src/agentforge/setup_files/.agentforge/settings/models.yaml new file mode 100644 index 00000000..f69403fe --- /dev/null +++ b/src/agentforge/setup_files/.agentforge/settings/models.yaml @@ -0,0 +1,123 @@ +# Default model settings for all agents unless overridden +default_model: + api: gemini_api + model: gemini_flash +# Uncomment to use alternative default models +# api: lm_studio_api +# model: LMStudio +# api: openai_api +# model: omni_model +# model: o1_preview + +# Library of models and parameter defaults +model_library: + openai_api: # API script name + O1Series: # Class name for the API (Case Sensitive) + models: # List of model configurations + o1: + identifier: o1 + params: {} # No overrides for this model + o1_preview: + identifier: o1-preview + o1_mini: + identifier: o1-mini + + params: {} # Default parameters for the model class + + GPT: + models: + omni_model: + identifier: gpt-4o + params: + max_new_tokens: 15000 # Example of overriding parameters + smart_model: + identifier: gpt-4 + smart_fast_model: + identifier: gpt-4-turbo + fast_model: + identifier: gpt-3.5-turbo + + params: # Default parameters for GPT models + max_tokens: 10000 + n: 1 + presence_penalty: 0 + stop: null + temperature: 0.8 + top_p: 0.1 + + anthropic_api: + Claude: + models: + claude3: + identifier: claude-3-opus-20240229 + + params: # Default parameters for Claude models + max_tokens: 10000 + temperature: 0.8 + top_p: 0.1 + + gemini_api: + Gemini: + models: + gemini_pro: + identifier: gemini-1.5-pro + gemini_flash: + identifier: gemini-1.5-flash + + params: # Default parameters for Gemini models + candidate_count: 1 + max_output_tokens: 10000 + temperature: 0.8 + top_k: 40 + top_p: 0.1 + + lm_studio_api: + LMStudio: + models: + llama3_8b: + identifier: lmstudio-community/Meta-Llama-3-8B-Instruct-GGUF + + params: # Default parameters for LMStudio models + host_url: http://localhost:1234/v1/chat/completions + max_tokens: 10000 + stream: false + temperature: 0.8 + + ollama_api: + Ollama: + models: + llama3.1_70b: + identifier: llama3.1:70b + + params: # Default parameters for Ollama models + host_url: http://localhost:11434/api/generate + max_tokens: 10000 + stream: false + temperature: 0.8 + + openrouter_api: + OpenRouter: + models: + phi3med: + identifier: microsoft/phi-3-medium-128k-instruct:free + hermes: + identifier: nousresearch/hermes-3-llama-3.1-405b + reflection: + identifier: mattshumer/reflection-70b:free + + groq_api: + GroqAPI: + models: + llama31: + identifier: llama-3.1-70b-versatile + + params: # Default parameters for GroqAPI models + max_tokens: 10000 + seed: -1 + stop: null + temperature: 0.8 + top_p: 0.1 + +# Embedding library +embedding_library: + library: sentence_transformers diff --git a/src/agentforge/setup_files/.agentforge/settings/storage.yaml b/src/agentforge/setup_files/.agentforge/settings/storage.yaml new file mode 100644 index 00000000..f5a90130 --- /dev/null +++ b/src/agentforge/setup_files/.agentforge/settings/storage.yaml @@ -0,0 +1,18 @@ +# Default storage settings for all agents unless overridden +options: + enabled: true + save_memory: true # Note: Saving memory won't work if storage is disabled + iso_timestamp: true # Use ISO format for timestamps + unix_timestamp: true # Use Unix format for timestamps + persist_directory: ./db/ChromaDB # Relative path for persistent storage + fresh_start: false # Wipes storage on system initialization if true + +# Selected Embedding +embedding: + selected: distil_roberta + +# Embedding library (mapping of embeddings to their identifiers) +embedding_library: + distil_roberta: all-distilroberta-v1 + all_mini: all-MiniLM-L6-v2 + diff --git a/src/agentforge/setup_files/.agentforge/settings/system.yaml b/src/agentforge/setup_files/.agentforge/settings/system.yaml new file mode 100644 index 00000000..8caa4252 --- /dev/null +++ b/src/agentforge/setup_files/.agentforge/settings/system.yaml @@ -0,0 +1,27 @@ +# Persona settings +persona: + enabled: true + name: default + +# Debug settings +debug: + mode: false + save_memory: false # Save memory during debugging (overrides normal behavior) + simulated_response: "Text designed to simulate an LLM response for debugging purposes without invoking the model." + +# Logging settings +logging: + enabled: true + console_level: warning + folder: ./logs + files: # Log levels: critical, error, warning, info, debug + agentforge: error + model_io: error + +# Miscellaneous settings +misc: + on_the_fly: true # Enables real-time dynamic adjustments for prompts and agent setting overrides + +# System file paths (Read/Write access) +paths: + files: ./files diff --git a/src/agentforge/setup_files/tools/__init__.py b/src/agentforge/setup_files/.agentforge/tools/__init__.py similarity index 100% rename from src/agentforge/setup_files/tools/__init__.py rename to src/agentforge/setup_files/.agentforge/tools/__init__.py diff --git a/src/agentforge/setup_files/.agentforge/tools/brave_search.yaml b/src/agentforge/setup_files/.agentforge/tools/brave_search.yaml new file mode 100644 index 00000000..65dcca26 --- /dev/null +++ b/src/agentforge/setup_files/.agentforge/tools/brave_search.yaml @@ -0,0 +1,35 @@ +Name: Brave Search +Args: + - query (str) + - count (int, optional) +Command: search +Description: |- + The 'Brave Search' tool performs a web search using the Brave Search API. It retrieves search results based on the provided query. Each result includes the title, URL, description, and any extra snippets. + +Instruction: |- + To use the 'Brave Search' tool, follow these steps: + 1. Call the `search` method with the following arguments: + - `query`: A string representing the search query. + - `count`: (Optional) An integer specifying the number of search results to retrieve. Defaults to 10 if not specified. + 2. The method returns a dictionary containing search results in the keys: + - `'web_results'`: A list of web search results. + - `'video_results'`: A list of video search results (if any). + 3. Each item in `'web_results'` includes: + - `title`: The title of the result. + - `url`: The URL of the result. + - `description`: A brief description of the result. + - `extra_snippets`: (Optional) Additional snippets of information. + 4. Utilize the returned results as needed in your application. + +Example: |- + # Example usage of the Brave Search tool: + brave_search = BraveSearch() + results = brave_search.search(query='OpenAI GPT-4', count=5) + for result in results['web_results']: + print(f"Title: {result['title']}") + print(f"URL: {result['url']}") + print(f"Description: {result['description']}") + print('---') + +Script: .agentforge.tools.brave_search +Class: BraveSearch diff --git a/src/agentforge/setup_files/.agentforge/tools/file_writer.yaml b/src/agentforge/setup_files/.agentforge/tools/file_writer.yaml new file mode 100755 index 00000000..c292221a --- /dev/null +++ b/src/agentforge/setup_files/.agentforge/tools/file_writer.yaml @@ -0,0 +1,14 @@ +Name: File Writer +Args: + - folder (str) + - file (str) + - text (str) + - mode (str='a') +Command: write_file +Description: >- + The 'File Writer' tool writes the provided text to a specified file. You can specify the folder, filename, and the mode (append or overwrite). +Example: response = write_file(folder, file, text, mode='a') +Instruction: >- + The 'write_file' method requires a folder, file name, and the text you want to write as inputs. An optional mode parameter can be provided to decide whether to append ('a') or overwrite ('w') the file. By default, the function appends to the file. +Script: .agentforge.tools.write_file +Class: WriteFile diff --git a/src/agentforge/setup_files/tools/GoogleSearch.yaml b/src/agentforge/setup_files/.agentforge/tools/google_search.yaml similarity index 92% rename from src/agentforge/setup_files/tools/GoogleSearch.yaml rename to src/agentforge/setup_files/.agentforge/tools/google_search.yaml index c52fa5de..30227ef3 100755 --- a/src/agentforge/setup_files/tools/GoogleSearch.yaml +++ b/src/agentforge/setup_files/.agentforge/tools/google_search.yaml @@ -11,4 +11,4 @@ Instruction: >- The 'google_search' function takes a query string and a number of results as inputs. The query string is what you want to search for, and the number of results is how many search results you want returned. The function returns a list of tuples, each tuple containing a URL and a snippet description of a search result. -Script: .agentforge.tools.GoogleSearch +Script: .agentforge.tools.google_search diff --git a/Sandbox/.agentforge/tools/IntelligentChunk.yaml b/src/agentforge/setup_files/.agentforge/tools/intelligent_chunk.yaml similarity index 92% rename from Sandbox/.agentforge/tools/IntelligentChunk.yaml rename to src/agentforge/setup_files/.agentforge/tools/intelligent_chunk.yaml index 7a736700..452f3ae7 100755 --- a/Sandbox/.agentforge/tools/IntelligentChunk.yaml +++ b/src/agentforge/setup_files/.agentforge/tools/intelligent_chunk.yaml @@ -8,4 +8,4 @@ Description: >- Example: chunks = intelligent_chunk(text, chunk_size) Instruction: >- The 'intelligent_chunk' method takes a string of text and a chunk size as inputs. The chunk size is an integer that determines the number of sentences per chunk: 0 for 5 sentences, 1 for 13 sentences, 2 for 34 sentences, and 3 for 55 sentences. The function returns a list of text chunks, each containing a specified number of sentences. -Script: .agentforge.tools.IntelligentChunk +Script: .agentforge.tools.intelligent_chunk diff --git a/src/agentforge/setup_files/.agentforge/tools/read_directory.yaml b/src/agentforge/setup_files/.agentforge/tools/read_directory.yaml new file mode 100755 index 00000000..99eec434 --- /dev/null +++ b/src/agentforge/setup_files/.agentforge/tools/read_directory.yaml @@ -0,0 +1,17 @@ +Name: Read Directory +Args: + - directory_paths (str or list of str) + - max_depth (int, optional) +Command: read_directory +Description: >- + The 'Read Directory' tool prints the structure of a directory or multiple directories in a tree-like format. It visually represents folders and files, and you can specify the depth of the structure to be printed. The tool can handle both a single directory path or a list of directory paths. If a specified path does not exist, the tool will create it. Additionally, it indicates if a directory is empty or if there are more files beyond the specified depth. +Example: >- + # For a single directory + directory_structure = read_directory('/path/to/directory', max_depth=3) + + # For multiple directories + directory_structure = read_directory(['/path/to/directory1', '/path/to/directory2'], max_depth=2) +Instruction: >- + The 'read_directory' method requires either a single directory path (string) or a list of directory paths (list of strings). An optional max_depth parameter can be provided to limit the depth of the directory structure displayed. The method returns a string representing the directory structure. It handles directory creation if the path does not exist and checks if directories are empty. The method includes error handling for permissions and file not found errors. +Script: .agentforge.tools.directory +Class: Directory diff --git a/Sandbox/.agentforge/tools/ReadFile.yaml b/src/agentforge/setup_files/.agentforge/tools/read_file.yaml similarity index 90% rename from Sandbox/.agentforge/tools/ReadFile.yaml rename to src/agentforge/setup_files/.agentforge/tools/read_file.yaml index 19fcb554..140277f0 100755 --- a/Sandbox/.agentforge/tools/ReadFile.yaml +++ b/src/agentforge/setup_files/.agentforge/tools/read_file.yaml @@ -6,4 +6,4 @@ Description: >- Example: file_content = read_file(file_path) Instruction: >- The 'read_file' method requires a file_path as input, which represents the path to the file you want to read. It returns the textual content of that file as a string. -Script: .agentforge.tools.ReadFile +Script: .agentforge.tools.read_file diff --git a/src/agentforge/setup_files/.agentforge/tools/web_scrape.yaml b/src/agentforge/setup_files/.agentforge/tools/web_scrape.yaml new file mode 100755 index 00000000..de044d6c --- /dev/null +++ b/src/agentforge/setup_files/.agentforge/tools/web_scrape.yaml @@ -0,0 +1,10 @@ +Name: Web Scrape +Args: url (str) +Command: get_plain_text +Description: >- + The 'Web Scrape' tool is used to pull all text from a webpage. Simply provide the web address (URL), and the tool will return the webpage's content in plain text. +Example: scrapped = get_plain_text(url) +Instruction: >- + The 'get_plain_text' method of the 'Web Scrape' tool takes a URL as an input, which represents the webpage to scrape. It returns the textual content of that webpage as a string. You can send only one URL, so if you receive more than one, choose the most likely URL to contain the results you expect. +Script: .agentforge.tools.web_scrape +Class: WebScrape diff --git a/src/agentforge/setup_files/prompts/modules/LearnKGAgent.yaml b/src/agentforge/setup_files/prompts/modules/LearnKGAgent.yaml deleted file mode 100644 index 9ff8b452..00000000 --- a/src/agentforge/setup_files/prompts/modules/LearnKGAgent.yaml +++ /dev/null @@ -1,36 +0,0 @@ -Prompts: - System: - Task: |+ - You are an advanced text analysis agent with a specific focus on enhancing knowledge graphs. Your task involves meticulously parsing through given text to identify and extract sentences containing new, significant information. This information will be integrated into a knowledge graph to augment the intelligence of AI systems. Be mindful that efficiency is key; unnecessary duplication of existing knowledge is to be avoided, except when the knowledge graph is initially empty. - - ExistingKnowledge: |+ - The knowledge graph currently contains these entries. Ensure that new selections offer distinct and valuable information, unless the knowledge graph is empty, in which case, prioritize capturing foundational knowledge: - - ``` - {existing_knowledge} - ``` - - User: - Chunk: |+ - Analyze this text to find new and important knowledge: - - ``` - {text_chunk} - ``` - - Instruction: |+ - Examine the text chunk and select sentences that provide unique and substantial information. Your selections should fill gaps in the existing knowledge graph, avoiding redundancy. If the knowledge graph is empty, focus on identifying sentences that lay a foundational understanding. If no new relevant information is found, it's acceptable to select none. For each sentence chosen, explain why it's important and distinct from the current knowledge graph entries (or foundational in case of an empty knowledge graph). - - Make sure to include the code block triple backticks in your response for proper markdown format. Adhere strictly to the provided YAML response template. Include your selections and reasons within this format, refraining from any additional commentary. Only one line per sentence. - - RESPONSE TEMPLATE: - ```yaml - sentences: - # If any new and important sentences are identified, list them here. Otherwise, leave blank. - 1: - # Add more sentences if necessary. - reasons: - # Corresponding reasons for each selected sentence. If no sentences are selected, leave blank. - 1: - # Continue with reasons for additional sentences if there are any. - ``` \ No newline at end of file diff --git a/src/agentforge/setup_files/prompts/modules/MetadataKGAgent.yaml b/src/agentforge/setup_files/prompts/modules/MetadataKGAgent.yaml deleted file mode 100644 index ede0e2dc..00000000 --- a/src/agentforge/setup_files/prompts/modules/MetadataKGAgent.yaml +++ /dev/null @@ -1,37 +0,0 @@ -Prompts: - System: - Task: |+ - You are an advanced text analysis agent with a specific focus on enhancing knowledge graphs. Your core responsibility is to extract and structure metadata from provided text to enrich a knowledge graph, enhancing the system's intelligence. Aim for efficiency and precision, avoiding the duplication of information unless dealing with an empty knowledge graph. - - ExistingKnowledge: |+ - Given the knowledge graph's current entries, your task is to augment it with new and distinct metadata derived from the provided sentence and its contextual background: - - ``` - {existing_knowledge} - ``` - User: - Context: |+ - Context paragraph: - - ``` - {text_chunk} - ``` - - Sentence: |+ - Sentence: - - ``` - {sentence} - ``` - - Instructions: |+ - Analyze the given sentence within the context provided. Generate metadata that contributes unique and valuable insights to the knowledge graph. Ensure your entries connect to existing graph data when applicable, avoiding redundancy. If the knowledge graph is initially empty, emphasize establishing a solid foundational layer of information. Your output should specify the subject, predicate, and object from the sentence and include at least three additional relevant metadata tags, each as a separate entry without using lists or leading underscores. - - Use the following YAML response template and adhere strictly to its structure. Your response should be wrapped in a yaml code block. Include the necessary metadata and corresponding rationale within this format without any extra commentary: - - ```yaml - subject: - predicate: - object: - : - ``` \ No newline at end of file diff --git a/src/agentforge/setup_files/settings/models.yaml b/src/agentforge/setup_files/settings/models.yaml deleted file mode 100644 index 01a9e64f..00000000 --- a/src/agentforge/setup_files/settings/models.yaml +++ /dev/null @@ -1,112 +0,0 @@ -# Default settings for all models unless overridden -ModelSettings: - API: openai_api - Model: omni_model -# API: lm_studio_api -# Model: LMStudio - Params: # Default parameter values - max_new_tokens: 3000 - temperature: 0.8 - top_p: 0.1 - n: 1 - stop: null - do_sample: true - return_prompt: false - return_metadata: false - typical_p: 0.95 - repetition_penalty: 1.05 - encoder_repetition_penalty: 1.0 - top_k: 40 - min_length: 10 - no_repeat_ngram_size: 0 - num_beams: 1 - penalty_alpha: 0 - length_penalty: 1 - early_stopping: false - pad_token_id: null - eos_token_id: null - use_cache: true - num_return_sequences: 1 - bad_words_ids: null - seed: -1 - -# Library of Models and Parameter Defaults Override -ModelLibrary: - openai_api: - module: "openai" - class: "GPT" - models: - omni_model: - name: gpt-4o - params: # Specific parameters for the model - max_new_tokens: 3500 - smart_model: - name: gpt-4 - smart_fast_model: - name: gpt-4-turbo-2024-04-09 - fast_model: - name: gpt-3.5-turbo - long_fast_model: - name: gpt-3.5-turbo-16k - old_fast_model: - name: gpt-3.5-turbo-0613 - old_long_fast_model: - name: gpt-3.5-turbo-16k-0613 - groq_api: - module: "groq_api" - class: "GroqAPI" - models: - llama31: - name: llama-3.1-70b-versatile - openrouter_api: - module: "openrouter" - class: "OpenRouter" - models: - phi3med: - name: microsoft/phi-3-medium-128k-instruct:free - hermes: - name: nousresearch/hermes-3-llama-3.1-405b - reflection: - name: mattshumer/reflection-70b:free - claude_old: - module: "claude_old" - class: "Claude" - models: - claude: - name: claude-2 - claude3_api: - module: "anthropic" - class: "Claude" - models: - claude-3: - name: claude-3-opus-20240229 - gemini_api: - module: "gemini" - class: "Gemini" - models: - gemini-pro: - name: gemini-1.5-pro - gemini-flash: - name: gemini-1.5-flash - lm_studio_api: - module: "LMStudio" - class: "LMStudio" - models: - LMStudio: - name: lmstudio-community/Meta-Llama-3-8B-Instruct-GGUF - params: - host_url: "http://localhost:1234/v1/chat/completions" - allow_custom_value: True - ollama_api: - module: "ollama" - class: "Ollama" - models: - Llama3.1_70b: - name: "llama3.1:70b" - params: - host_url: "http://localhost:11434/api/generate" - allow_custom_value: True - -# Embedding Library (Not much to see here) -EmbeddingLibrary: - library: sentence_transformers diff --git a/src/agentforge/storage/__init__.py b/src/agentforge/storage/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/agentforge/utils/ChromaUtils.py b/src/agentforge/storage/chroma_storage.py similarity index 87% rename from src/agentforge/utils/ChromaUtils.py rename to src/agentforge/storage/chroma_storage.py index 00fbb32e..b6f837b1 100755 --- a/src/agentforge/utils/ChromaUtils.py +++ b/src/agentforge/storage/chroma_storage.py @@ -1,19 +1,16 @@ import os import uuid -# from pathlib import Path from datetime import datetime from typing import Optional, Union -# from scipy.spatial import distance - import chromadb from chromadb.config import Settings from chromadb.utils import embedding_functions -from agentforge.utils.Logger import Logger -from ..config import Config +from agentforge.utils.logger import Logger +from agentforge.config import Config -logger = Logger(name="Chroma Utils") +logger = Logger(name="Chroma Utils", default_logger='chroma_utils') os.environ["TOKENIZERS_PARALLELISM"] = "false" @@ -61,26 +58,32 @@ def generate_defaults(data: Union[list, str], ids: list = None, metadata: list[d return ids, metadata -def apply_timestamps(metadata: list[dict], config): - """ - Applies timestamps to the metadata if required by the configuration. - - Parameters: - metadata (list[dict]): The metadata for the documents. - config (dict): The configuration dictionary. - """ - do_time_stamp = config['settings']['system'].get('ISOTimeStampMemory') +def apply_iso_timestamps(metadata: list[dict], config): + do_time_stamp = config['settings']['storage']['options'].get('iso_timestamp', False) if do_time_stamp: timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S') for m in metadata: - m['isotimestamp'] = timestamp + m['iso_timestamp'] = timestamp + - do_time_stamp = config['settings']['system'].get('UnixTimeStampMemory') +def apply_unix_timestamps(metadata: list[dict], config): + do_time_stamp = config['settings']['storage']['options'].get('unix_timestamp', False) if do_time_stamp: timestamp = datetime.now().timestamp() for m in metadata: - m['unixtimestamp'] = timestamp + m['unix_timestamp'] = timestamp + + +def apply_timestamps(metadata: list[dict], config): + """ + Applies timestamps to the metadata if required by the configuration. + Parameters: + metadata (list[dict]): The metadata for the documents. + config (dict): The configuration dictionary. + """ + apply_iso_timestamps(metadata, config) + apply_unix_timestamps(metadata, config) def save_to_collection(collection, data: list, ids: list, metadata: list[dict]): """ @@ -99,7 +102,7 @@ def save_to_collection(collection, data: list, ids: list, metadata: list[dict]): ) -class ChromaUtils: +class ChromaStorage: """ A utility class for managing interactions with ChromaDB, offering a range of functionalities including initialization, data insertion, query, and collection management. @@ -115,47 +118,9 @@ class ChromaUtils: db_embed = None embedding = None - def __init__(self, persona_name="default"): - """ - Ensures an instance of ChromaUtils is created. Initializes embeddings and storage - upon creation. - """ - self.persona_name = persona_name - self.config = Config() - self.init_embeddings() - self.init_storage() - - def init_embeddings(self): - """ - Initializes the embedding function based on the configuration, supporting multiple embedding backends. - - Raises: - KeyError: If a required environment variable or setting is missing. - Exception: For any errors that occur during the initialization of embeddings. - """ - try: - self.db_path, self.db_embed = self.chromadb_settings() - - # Initialize embedding based on the specified backend in the configuration - if self.db_embed == 'openai_ada2': - openai_api_key = os.getenv('OPENAI_API_KEY') - self.embedding = embedding_functions.OpenAIEmbeddingFunction( - api_key=openai_api_key, - model_name="text-embedding-ada-002" - ) - elif self.db_embed == 'all-distilroberta-v1': - self.embedding = embedding_functions.SentenceTransformerEmbeddingFunction( - model_name="all-distilroberta-v1") - # Additional embeddings can be initialized here similarly - else: - self.embedding = embedding_functions.SentenceTransformerEmbeddingFunction( - model_name="all-MiniLM-L12-v2") - except KeyError as e: - logger.log(f"Missing environment variable or setting: {e}", 'error') - raise - except Exception as e: - logger.log(f"Error initializing embeddings: {e}", 'error') - raise + # --------------------------------- + # Completed + # --------------------------------- def init_storage(self): """ @@ -172,12 +137,29 @@ def init_storage(self): else: self.client = chromadb.EphemeralClient() - if self.config.data['settings']['system'].get('DBFreshStart'): + if self.config.data['settings']['storage'].get('fresh_start'): self.reset_memory() except Exception as e: - logger.log(f"Error initializing storage: {e}", 'error') + logger.log(f"[init_storage] Error initializing storage: {e}", 'error') raise + def select_collection(self, collection_name: str): + """ + Selects (or creates if not existent) a collection within the storage by name. + + Parameters: + collection_name (str): The name of the collection to select or create. + + Raises: + ValueError: If there's an error in getting or creating the collection. + """ + try: + self.collection = self.client.get_or_create_collection(name=collection_name, + embedding_function=self.embedding, + metadata={"hnsw:space": "cosine"}) + except Exception as e: + raise ValueError(f"\n\nError getting or creating collection. Error: {e}") + def chromadb_settings(self): """ Retrieves the ChromaDB settings from the configuration. @@ -186,11 +168,11 @@ def chromadb_settings(self): tuple: A tuple containing the database path and embedding settings. """ # Retrieve the ChromaDB settings - sys_settings = self.config.data['settings']['system'] + storage_settings = self.config.data['settings']['storage'] # Get the database path and embedding settings - db_path_setting = sys_settings.get('PersistDirectory', None) - db_embed = sys_settings.get('Embedding', None) + db_path_setting = storage_settings['options'].get('persist_directory', None) + db_embed = storage_settings['embedding'].get('selected', None) # Construct the absolute path of the database using the project root if db_path_setting: @@ -201,22 +183,51 @@ def chromadb_settings(self): return db_path, db_embed - def select_collection(self, collection_name: str): + def init_embeddings(self): """ - Selects (or creates if not existent) a collection within the storage by name. - - Parameters: - collection_name (str): The name of the collection to select or create. + Initializes the embedding function based on the configuration, supporting multiple embedding backends. Raises: - ValueError: If there's an error in getting or creating the collection. + KeyError: If a required environment variable or setting is missing. + Exception: For any errors that occur during the initialization of embeddings. """ try: - self.collection = self.client.get_or_create_collection(name=collection_name, - embedding_function=self.embedding, - metadata={"hnsw:space": "cosine"}) + self.db_path, self.db_embed = self.chromadb_settings() + + # Initialize embedding based on the specified backend in the configuration + if self.db_embed == 'openai_ada2': + openai_api_key = os.getenv('OPENAI_API_KEY') + self.embedding = embedding_functions.OpenAIEmbeddingFunction( + api_key=openai_api_key, + model_name="text-embedding-ada-002" + ) + elif self.db_embed == 'all-distilroberta-v1': + self.embedding = embedding_functions.SentenceTransformerEmbeddingFunction( + model_name="all-distilroberta-v1") + # Additional embeddings can be initialized here similarly + else: + self.embedding = embedding_functions.SentenceTransformerEmbeddingFunction( + model_name="all-MiniLM-L12-v2") + except KeyError as e: + logger.log(f"[init_embeddings] Missing environment variable or setting: {e}", 'error') + raise except Exception as e: - raise ValueError(f"\n\nError getting or creating collection. Error: {e}") + logger.log(f"[init_embeddings] Error initializing embeddings: {e}", 'error') + raise + + # --------------------------------- + # Pending + # --------------------------------- + + def __init__(self, persona_name="default"): + """ + Ensures an instance of ChromaUtils is created. Initializes embeddings and storage + upon creation. + """ + self.persona_name = persona_name + self.config = Config() + self.init_embeddings() + self.init_storage() def delete_collection(self, collection_name: str): """ @@ -321,8 +332,8 @@ def save_memory(self, collection_name: str, data: Union[list, str], ids: list = during the save operation. """ - if self.config.data['settings']['system']['SaveMemory'] is False: - print("\nMemory Saving is Disabled. To Enable Memory Saving, set the 'SaveMemory' flag to 'true' in the " + if self.config.data['settings']['storage']['options']['save_memory'] is False: + print("\nMemory Saving is Disabled. To Enable Memory Saving, set the 'save_memory' flag to 'true' in the " "system.yaml file.\n") return @@ -340,7 +351,7 @@ def save_memory(self, collection_name: str, data: Union[list, str], ids: list = save_to_collection(self.collection, data, ids, metadata) except Exception as e: - raise ValueError(f"Error saving results. Error: {e}\n\nData:\n{data}") + raise ValueError(f"[Chroma Utils] [save_memory] Error saving results. Error: {e}\n\nData:\n{data}") def query_memory(self, collection_name: str, query: Optional[Union[str, list]] = None, filter_condition: Optional[dict] = None, include: Optional[list] = None, @@ -408,9 +419,11 @@ def query_memory(self, collection_name: str, query: Optional[Union[str, list]] = return result except Exception as e: - logger.log(f"Error querying memory: {e}", 'error') + logger.log(f"[query_memory] Error querying memory: {e}", 'error') return None + # Done + def reset_memory(self): """ Resets the entire storage, removing all collections and their data. @@ -462,14 +475,14 @@ def search_storage_by_threshold(self, collection_name: str, query: str, threshol if filtered_data['documents']: return filtered_data else: - logger.log('Search by Threshold: No documents found that meet the threshold.', 'info') + logger.log('[search_storage_by_threshold] No documents found that meet the threshold.', 'info') else: logger.log('Search by Threshold: No documents found.', 'info') return {} except Exception as e: - logger.log(f"Error searching storage by threshold: {e}", 'error') + logger.log(f"[search_storage_by_threshold] Error searching storage by threshold: {e}", 'error') return {'failed': f"Error searching storage by threshold: {e}"} def return_embedding(self, text_to_embed: str): @@ -522,7 +535,7 @@ def search_metadata_min_max(self, collection_name, metadata_tag, min_max): # Check if all metadata values are numeric (int or float) if not all(isinstance(value, (int, float)) for value in metadata_values): - logger.log(f"Error: The metadata tag '{metadata_tag}' contains non-numeric values.", 'error') + logger.log(f"[search_metadata_min_max] Error: The metadata tag '{metadata_tag}' contains non-numeric values.", 'error') return None if metadata_values: @@ -532,7 +545,7 @@ def search_metadata_min_max(self, collection_name, metadata_tag, min_max): try: target_index = metadata_values.index(max(metadata_values)) except: - logger.log(f"Error: The metadata tag '{metadata_tag}' is empty or does not exist. Returning 0.", 'error') + logger.log(f"[search_metadata_min_max] Error: The metadata tag '{metadata_tag}' is empty or does not exist. Returning 0.", 'error') target_index = 0 else: target_index = 0 @@ -549,7 +562,7 @@ def search_metadata_min_max(self, collection_name, metadata_tag, min_max): } logger.log( - f"Found the following record by max value of {metadata_tag} metadata tag:\n{max_metadata}", + f"[search_metadata_min_max] Found the following record by max value of {metadata_tag} metadata tag:\n{max_metadata}", 'debug' ) return max_metadata @@ -557,7 +570,7 @@ def search_metadata_min_max(self, collection_name, metadata_tag, min_max): return None except (KeyError, ValueError, IndexError) as e: - logger.log(f"Error finding max metadata: {e}\nCollection: {collection_name}\nTarget Metadata: {metadata_tag}", 'error') + logger.log(f"[search_metadata_min_max] Error finding max metadata: {e}\nCollection: {collection_name}\nTarget Metadata: {metadata_tag}", 'error') return None def delete_memory(self, collection_name, doc_id): @@ -587,7 +600,7 @@ def rerank_results(self, query_results: dict, query: str, temp_collection_name: # Check if documents is empty if not query_results['documents']: - logger.log("No documents found in query_results. Skipping reranking.", 'warning') + logger.log("[rerank_results] No documents found in query_results. Skipping reranking.", 'warning') return query_results # Save the query results to a temporary collection @@ -616,10 +629,10 @@ def rerank_results(self, query_results: dict, query: str, temp_collection_name: return reranked_results except KeyError as e: - logger.log(f"KeyError occurred while reranking results: {e}", 'error') + logger.log(f"[rerank_results] KeyError occurred while reranking results: {e}", 'error') return None except Exception as e: - logger.log(f"Unexpected error occurred while reranking results: {e}", 'error') + logger.log(f"[rerank_results] Unexpected error occurred while reranking results: {e}", 'error') return None @staticmethod diff --git a/src/agentforge/tools/GetText.py b/src/agentforge/tools/GetText.py deleted file mode 100755 index df1a0e0a..00000000 --- a/src/agentforge/tools/GetText.py +++ /dev/null @@ -1,135 +0,0 @@ -import os -import requests -import io -import pypdf - - -class GetText: - def read_file(self, file_name_or_url: str) -> str: - """ - Reads text from a file or URL based on the given input. - - Parameters: - file_name_or_url (str): The file name or URL to read from. - - Returns: - str: The text content of the file or URL. - - Raises: - FileNotFoundError: If the file does not exist. - ValueError: If the file format is unsupported or if any error occurs. - """ - if file_name_or_url.startswith('http://') or file_name_or_url.startswith('https://'): - return self.read_from_url(file_name_or_url) - else: - if file_name_or_url.endswith('.pdf'): - return self.read_pdf(file_name_or_url) - elif file_name_or_url.endswith('.txt') or file_name_or_url.endswith('.md'): - return self.read_txt(file_name_or_url) - else: - raise ValueError("Unsupported file format - Use URL, PDF, TXT, or Markdown formats.") - - def read_pdf(self, filename: str) -> str: - """ - Reads text from a PDF file. - - Parameters: - filename (str): The path to the PDF file. - - Returns: - str: The text content of the PDF. - - Raises: - FileNotFoundError: If the file does not exist. - Exception: If any other error occurs during PDF reading. - """ - if not os.path.exists(filename): - raise FileNotFoundError("File not found") - - with open(filename, 'rb') as file: - content = io.BytesIO(file.read()) - return self.extract_text_from_pdf(content) - - @staticmethod - def read_txt(filename: str) -> str: - """ - Reads text from a TXT file. - - Parameters: - filename (str): The path to the TXT file. - - Returns: - str: The text content of the TXT file. - - Raises: - FileNotFoundError: If the file does not exist. - Exception: If any other error occurs during text file reading. - """ - if not os.path.exists(filename): - raise FileNotFoundError("File not found") - - try: - with open(filename, 'r', encoding='utf-8') as file: - text = file.read() - return text - except Exception as e: - raise Exception(f"Error reading TXT file: {str(e)}") - - def read_from_url(self, url: str) -> str: - """ - Reads text from a URL. - - Parameters: - url (str): The URL to read from. - - Returns: - str: The text content of the URL. - - Raises: - ValueError: If the file format is unsupported. - requests.RequestException: If any HTTP error occurs. - """ - try: - response = requests.get(url) - response.raise_for_status() - if url.endswith('.pdf'): - return self.extract_text_from_pdf(io.BytesIO(response.content)) - elif url.endswith('.txt'): - return response.text - else: - raise ValueError("Unsupported file format") - except requests.RequestException as e: - raise Exception(f"Error reading from URL: {str(e)}") - - @staticmethod - def extract_text_from_pdf(file_stream: io.BytesIO) -> str: - """ - Extracts text from a PDF file stream. - - Parameters: - file_stream (io.BytesIO): The file stream of the PDF. - - Returns: - str: The text content of the PDF. - - Raises: - Exception: If any error occurs during PDF text extraction. - """ - try: - text = "" - reader = pypdf.PdfReader(file_stream) - for page in reader.pages: - text += page.extract_text() - return text - except Exception as e: - raise Exception(f"Error extracting text from PDF: {str(e)}") - - -if __name__ == "__main__": - gettext_instance = GetText() - filename_or_url = 'Documents' # Replace with your file path or URL - try: - file_content = gettext_instance.read_file(filename_or_url) - print(file_content) - except Exception as exc: - print(f"An error occurred: {str(exc)}") diff --git a/src/agentforge/tools/SemanticChunk_old.py b/src/agentforge/tools/SemanticChunk_old.py deleted file mode 100644 index 1b867e2f..00000000 --- a/src/agentforge/tools/SemanticChunk_old.py +++ /dev/null @@ -1,82 +0,0 @@ -from semantic_chunkers import StatisticalChunker -from semantic_router.encoders import FastEmbedEncoder - -class Chunk: - def __init__(self, is_triggered, triggered_score, token_count, splits): - self.is_triggered = is_triggered - self.triggered_score = triggered_score - self.token_count = token_count - self.splits = splits - self.content = ' '.join(splits) # Join splits for content - -def semantic_chunk(text: str) -> list[Chunk]: - """ - Perform semantic chunking on the input text. - - This function uses a StatisticalChunker with a FastEmbedEncoder to split the input text - into semantically coherent chunks. It's designed to handle large texts by breaking them - down into smaller, meaningful segments. - - Args: - text (str): The input text to be chunked. - - Returns: - list[Chunk]: A list of Chunk objects, each representing a semantic chunk of the input text. - Each Chunk object has the following attributes: - - is_triggered (bool): Indicates if the chunk was triggered by the chunking algorithm. - - triggered_score (float): The score that triggered the chunk split. - - token_count (int): The number of tokens in the chunk. - - splits (list[str]): The individual text segments that make up the chunk. - - content (str): The full text content of the chunk (joined splits). - - Note: - This function uses the 'sentence-transformers/all-MiniLM-L6-v2' model for encoding. - The chunker is configured with specific parameters for token limits and window size, - which can be adjusted if needed. - """ - encoder = FastEmbedEncoder(name="sentence-transformers/all-MiniLM-L6-v2") - chunker = StatisticalChunker( - encoder=encoder, - dynamic_threshold=True, - min_split_tokens=128, - max_split_tokens=1024, - window_size=2, - enable_statistics=True # to print chunking stats - ) - - chunks = chunker([text]) - chunker.print(chunks[0]) - - result = [] - for chunk in chunks[0]: - chunk_obj = Chunk( - is_triggered=chunk.is_triggered, - triggered_score=chunk.triggered_score, - token_count=chunk.token_count, - splits=chunk.splits - ) - result.append(chunk_obj) - - return result - - - -if __name__ == '__main__': - import io - import requests - from PyPDF2 import PdfReader - - url = 'https://arxiv.org/pdf/2404.16811.pdf' - response = requests.get(url) - - if response.status_code == 200: - pdf_content = io.BytesIO(response.content) - pdf_reader = PdfReader(pdf_content) - text2 = "" - for page in pdf_reader.pages: - text2 += page.extract_text() - results = semantic_chunk(text2) - for r in results: - print(r.content) - else: - print(f"Failed to download the PDF. Status code: {response.status_code}") \ No newline at end of file diff --git a/src/agentforge/tools/BraveSearch.py b/src/agentforge/tools/brave_search.py similarity index 100% rename from src/agentforge/tools/BraveSearch.py rename to src/agentforge/tools/brave_search.py diff --git a/src/agentforge/tools/CleanString.py b/src/agentforge/tools/clean_string.py similarity index 100% rename from src/agentforge/tools/CleanString.py rename to src/agentforge/tools/clean_string.py diff --git a/src/agentforge/tools/CommandExecutor.py b/src/agentforge/tools/command_executor.py similarity index 100% rename from src/agentforge/tools/CommandExecutor.py rename to src/agentforge/tools/command_executor.py diff --git a/src/agentforge/tools/Directory.py b/src/agentforge/tools/directory.py similarity index 100% rename from src/agentforge/tools/Directory.py rename to src/agentforge/tools/directory.py diff --git a/src/agentforge/tools/get_text.py b/src/agentforge/tools/get_text.py new file mode 100755 index 00000000..4bd78d7d --- /dev/null +++ b/src/agentforge/tools/get_text.py @@ -0,0 +1,161 @@ +# get_text.py +import requests +import io +from pathlib import Path +import pypdf + + +class GetText: + @staticmethod + def resolve_path(filename: str) -> Path: + """ + Resolves a given path to an absolute path and validates its existence. + + Parameters: + filename (str): The file path to resolve. + + Returns: + Path: A resolved absolute path. + + Raises: + FileNotFoundError: If the file does not exist. + """ + path = Path(filename).expanduser().resolve() + + if not path.is_file(): + raise FileNotFoundError(f"{path}") + + return path + + def read_file(self, file_name_or_url: str) -> str: + """ + Reads text content from a file or URL. + + Parameters: + file_name_or_url (str): The path to the file or the URL to read. + + Returns: + str: The text content of the file or URL. + + Raises: + FileNotFoundError: If the file does not exist. + ValueError: If the file format is unsupported. + Exception: For general errors during reading. + """ + if file_name_or_url.startswith(('http://', 'https://')): + return self.read_from_url(file_name_or_url) + + if file_name_or_url.endswith('.pdf'): + return self.read_pdf(file_name_or_url) + + if file_name_or_url.endswith(('.txt', '.md')): + return self.read_txt(file_name_or_url) + + raise ValueError("Unsupported file format - Use a URL or File with PDF, TXT, or Markdown formats.") + + def read_pdf(self, filename: str) -> str: + """ + Reads text content from a PDF file. + + Parameters: + filename (str): The path to the PDF file. + + Returns: + str: The text content of the PDF. + + Raises: + FileNotFoundError: If the file does not exist. + Exception: For errors during PDF text extraction. + """ + path = self.resolve_path(filename) + + try: + with path.open('rb') as file: + content = io.BytesIO(file.read()) + return self.extract_text_from_pdf(content) + except Exception as e: + raise Exception(f"Error reading PDF file: {str(e)}") + + def read_txt(self, filename: str) -> str: + """ + Reads text content from a TXT file. + + Parameters: + filename (str): The path to the TXT file. + + Returns: + str: The text content of the TXT file. + + Raises: + FileNotFoundError: If the file does not exist. + Exception: For errors during TXT file reading. + """ + path = self.resolve_path(filename) + + try: + return path.read_text(encoding='utf-8') + except Exception as e: + raise Exception(f"Error reading TXT file: {str(e)}") + + def read_from_url(self, url: str) -> str: + """ + Reads text content from a URL. + + Parameters: + url (str): The URL to read from. + + Returns: + str: The text content of the URL. + + Raises: + ValueError: If the file format is unsupported. + requests.RequestException: For HTTP errors during URL reading. + """ + try: + response = requests.get(url) + response.raise_for_status() + + if url.endswith('.pdf'): + return self.extract_text_from_pdf(io.BytesIO(response.content)) + + if url.endswith('.txt') or url.endswith('.md'): + return response.text + + raise ValueError("Unsupported file format for URL - Use PDF, TXT or Markdown formats.") + except requests.RequestException as e: + raise Exception(f"Error reading from URL: {str(e)}") + + @staticmethod + def extract_text_from_pdf(file_stream: io.BytesIO) -> str: + """ + Extracts text content from a PDF file stream. + + Parameters: + file_stream (io.BytesIO): The file stream of the PDF. + + Returns: + str: The extracted text content of the PDF. + + Raises: + Exception: For errors during PDF text extraction. + """ + try: + text = "" + reader = pypdf.PdfReader(file_stream) + for page in reader.pages: + page_text = page.extract_text() + if page_text: + text += page_text + return text.strip() + except Exception as e: + raise Exception(f"Error extracting text from PDF: {str(e)}") + + +if __name__ == "__main__": + gettext_instance = GetText() + filename_or_url = 'Documents/sample.pdf' # Replace with your file path or URL + try: + file_content = gettext_instance.read_file(filename_or_url) + print(file_content) + except Exception as exc: + print(f"An error occurred: {str(exc)}") diff --git a/src/agentforge/tools/GoogleSearch.py b/src/agentforge/tools/google_search.py similarity index 100% rename from src/agentforge/tools/GoogleSearch.py rename to src/agentforge/tools/google_search.py diff --git a/src/agentforge/tools/ImageToTxt.py b/src/agentforge/tools/image_to_txt.py similarity index 100% rename from src/agentforge/tools/ImageToTxt.py rename to src/agentforge/tools/image_to_txt.py diff --git a/src/agentforge/tools/IntelligentChunk.py b/src/agentforge/tools/intelligent_chunk.py similarity index 100% rename from src/agentforge/tools/IntelligentChunk.py rename to src/agentforge/tools/intelligent_chunk.py diff --git a/src/agentforge/tools/PythonFunction.py b/src/agentforge/tools/python_function.py similarity index 96% rename from src/agentforge/tools/PythonFunction.py rename to src/agentforge/tools/python_function.py index cad87a0e..851122ce 100644 --- a/src/agentforge/tools/PythonFunction.py +++ b/src/agentforge/tools/python_function.py @@ -1,6 +1,6 @@ -# tools/PythonFunction.py +# tools/python_function.py -from agentforge.utils.ToolUtils import ToolUtils +from agentforge.utils.tool_utils import ToolUtils class PythonFunction: diff --git a/src/agentforge/tools/SemanticChunk.py b/src/agentforge/tools/semantic_chunk.py similarity index 100% rename from src/agentforge/tools/SemanticChunk.py rename to src/agentforge/tools/semantic_chunk.py diff --git a/src/agentforge/tools/TripleExtract.py b/src/agentforge/tools/triple_extract.py similarity index 100% rename from src/agentforge/tools/TripleExtract.py rename to src/agentforge/tools/triple_extract.py diff --git a/src/agentforge/tools/UserInput.py b/src/agentforge/tools/user_input.py similarity index 100% rename from src/agentforge/tools/UserInput.py rename to src/agentforge/tools/user_input.py diff --git a/src/agentforge/tools/WebScrape.py b/src/agentforge/tools/web_scrape.py similarity index 85% rename from src/agentforge/tools/WebScrape.py rename to src/agentforge/tools/web_scrape.py index 1babc980..b29d9bde 100755 --- a/src/agentforge/tools/WebScrape.py +++ b/src/agentforge/tools/web_scrape.py @@ -2,10 +2,9 @@ import re from bs4 import BeautifulSoup -from ..tools.IntelligentChunk import intelligent_chunk -from ..utils.ChromaUtils import ChromaUtils +from agentforge.storage.chroma_storage import ChromaStorage -storage_instance = ChromaUtils() # Create an instance of ChromaUtils +storage_instance = ChromaStorage() # Create an instance of ChromaUtils def remove_extra_newlines(chunk): @@ -37,7 +36,7 @@ def get_plain_text(url): url (str): The URL of the webpage to scrape. Returns: - str: A message indicating that the webpage was saved to memory. + str: Plain text retrieved from the URL. Raises: ValueError: If the URL is not a string or is empty. @@ -57,10 +56,10 @@ def get_plain_text(url): # Extract the plain text from the HTML content plain_text = soup.get_text() - chunk_text = intelligent_chunk(plain_text, chunk_size=1) - chunk_save(chunk_text, url) + # chunk_text = intelligent_chunk(plain_text, chunk_size=1) + # chunk_save(chunk_text, url) - return f"Webpage saved to memory!\nURL: {url}" + return plain_text except requests.RequestException as e: raise Exception(f"Error fetching the webpage: {str(e)}") diff --git a/src/agentforge/tools/WriteFile.py b/src/agentforge/tools/write_file.py similarity index 100% rename from src/agentforge/tools/WriteFile.py rename to src/agentforge/tools/write_file.py diff --git a/src/agentforge/utils/DiscordClient.py b/src/agentforge/utils/DiscordClient.py deleted file mode 100644 index 760838e4..00000000 --- a/src/agentforge/utils/DiscordClient.py +++ /dev/null @@ -1,599 +0,0 @@ -# utils/DiscordClient.py - -import discord -import os -import asyncio -import threading -from agentforge.utils.Logger import Logger -from agentforge.tools.SemanticChunk import semantic_chunk - -class DiscordClient: - """ - A Discord client that handles bot functionality, message processing, and role management. - - This class uses a combination of asyncio and threading to manage Discord operations: - - The Discord client runs in a separate thread to avoid blocking the main application. - - Asynchronous methods are used for Discord API calls, which are then run in the client's event loop. - - Thread-safe methods are provided for external code to interact with the Discord client. - - Attributes: - token (str): The Discord bot token. - intents (discord.Intents): The intents for the Discord client. - client (discord.Client): The main Discord client instance. - logger (Logger): A custom logger for the Discord client. - tree (discord.app_commands.CommandTree): The command tree for slash commands. - message_queue (dict): A queue to store incoming messages, keyed by channel ID. - running (bool): A flag indicating whether the client is running. - discord_thread (threading.Thread): The thread running the Discord client. - """ - - def __init__(self): - """ - Initialize the DiscordClient with necessary attributes and event handlers. - """ - self.token = str(os.getenv('DISCORD_TOKEN')) - self.intents = discord.Intents.default() - self.intents.message_content = True - self.client = discord.Client(intents=self.intents) - self.logger = Logger('DiscordClient') - self.tree = discord.app_commands.CommandTree(self.client) - self.message_queue = {} - self.running = False - self.load_commands() - - @self.client.event - async def on_ready(): - await self.tree.sync() - print("Client Ready") - self.logger.log(f'{self.client.user} has connected to Discord!', 'info', 'DiscordClient') - - @self.client.event - async def on_message(message: discord.Message): - self.logger.log(f"On Message: {message}", 'debug', 'DiscordClient') - - content = message.content - for mention in message.mentions: - # If a mention is copy/pasted, this does not work. The mention value will come through as Null. - content = content.replace(f'<@{mention.id}>', f'@{mention.display_name}') - - message_data = { - "channel": str(message.channel), - "channel_id": message.channel.id, - "message": content, - "message_id": message.id, - "author": message.author.display_name, - "author_id": message.author, - "timestamp": message.created_at.strftime('%Y-%m-%d %H:%M:%S'), - "mentions": message.mentions - } - - # Add thread information to message_data if the message is in a thread - if isinstance(message.channel, discord.Thread): - message_data["thread_id"] = message.channel.id - message_data["thread_name"] = message.channel.name - - self.logger.log(f"{message.author.display_name} said: {content} in {str(message.channel)}. Channel ID: {message.channel.id}", 'info', 'DiscordClient') - # print(f"{author_name} said: {content} in {channel}. Channel ID: {channel_id}") - # print(f"Mentions: {formatted_mentions}") - - if message.author != self.client.user: - if message.channel.id not in self.message_queue: - self.message_queue[message.channel.id] = [] - self.message_queue[message.channel.id].append(message_data) - self.logger.log("Message added to queue", 'debug', 'DiscordClient') - else: - self.logger.log(f"Message not added to queue: {message_data}", 'debug', 'DiscordClient') - - def run(self): - """ - Start the Discord client in a separate thread. - - This method creates a new thread that runs the Discord client's event loop. - The thread allows the Discord client to operate independently of the main - application thread, preventing it from blocking other operations. - """ - def run_discord(): - print("Client Starting") - asyncio.run(self.client.start(self.token)) - - self.discord_thread = threading.Thread(target=run_discord) - self.discord_thread.start() - self.running = True - - def stop(self): - """ - Stop the Discord client and join the client thread. - - This method closes the Discord client's connection and waits for the - client thread to finish, ensuring a clean shutdown. - """ - self.running = False - asyncio.run(self.client.close()) - self.discord_thread.join() - - def process_channel_messages(self): - """ - Process and yield messages from the message queue. - - This function retrieves all messages sent to a discord channel from the - message_queue and yields them. - Each message is represented as a tuple with the following structure: - - (channel_id, [message_data]) - - where: - - channel_id (int): The ID of the Discord channel where the message was sent. - - message_data (list): A list containing a single dictionary with message details: - { - 'channel': str, # The name of the channel (e.g., 'system') - 'channel_id': int, # The ID of the channel (same as the tuple's first element) - 'message': str, # The content of the message - 'author': str, # The display name of the message author - 'author_id': Member, # The Discord Member object of the author - 'timestamp': str # The timestamp of the message in 'YYYY-MM-DD HH:MM:SS' format - 'mentions': list # A list of Discord Member objects mentioned in the message - } - - Yields: - tuple: A message tuple as described above. - - Note: - - This function is designed to work with Discord message objects. - - If the message queue is empty, the function will print "No Message Found" and pass. - - Any exceptions during message processing will be caught and printed. - """ - if self.message_queue: - try: - next_message = self.message_queue.popitem() - yield next_message - except Exception as e: - print(f"Exception: {e}") - else: - pass - - def send_message(self, channel_id, content): - """ - Send a message to a specified Discord channel. - - This method uses asyncio.run_coroutine_threadsafe to safely schedule the - asynchronous send operation in the Discord client's event loop, allowing - it to be called from any thread. - - Args: - channel_id (int): The ID of the channel to send the message to. - content (str): The content of the message to send. - """ - async def send(): - messages = semantic_chunk(content, min_length=200, max_length=1900) - channel = self.client.get_channel(channel_id) - if channel: - for msg in messages: - if len(msg.content) > 2000: - # Re-chunk the oversized message - sub_messages = semantic_chunk(msg.content, min_length=200, max_length=1900) - for sub_msg in sub_messages: - await channel.send(sub_msg.content) - else: - await channel.send(msg.content) - else: - self.logger.log(f"Channel {channel_id} not found", 'error', 'DiscordClient') - - asyncio.run_coroutine_threadsafe(send(), self.client.loop) - - def send_dm(self, user_id, content): - """ - Send a direct message to a specified Discord user. - - This method uses asyncio.run_coroutine_threadsafe to safely schedule the - asynchronous send operation in the Discord client's event loop, allowing - it to be called from any thread. - - Args: - user_id (int): The ID of the user to send the direct message to. - content (str): The content of the direct message to send. - """ - async def send_dm_async(): - try: - user = await self.client.fetch_user(user_id) - if user: - messages = semantic_chunk(content, min_length=200, max_length=1900) - for msg in messages: - if len(msg.content) > 2000: - # Re-chunk the oversized message - sub_messages = semantic_chunk(msg.content, min_length=200, max_length=1900) - for sub_msg in sub_messages: - await user.send(sub_msg.content) - else: - await user.send(msg.content) - else: - self.logger.log(f"User {user_id} not found", 'error', 'DiscordClient') - except discord.errors.NotFound: - self.logger.log(f"User {user_id} not found", 'error', 'DiscordClient') - except discord.errors.Forbidden: - self.logger.log(f"Cannot send DM to user {user_id}. Forbidden.", 'error', 'DiscordClient') - except Exception as e: - self.logger.log(f"Error sending DM to user {user_id}: {str(e)}", 'error', 'DiscordClient') - - asyncio.run_coroutine_threadsafe(send_dm_async(), self.client.loop) - - def send_embed(self, channel_id, title, fields, color='blue', image_url=None): - """ - Send an embed message to a specified Discord channel. - - This method uses asyncio.run_coroutine_threadsafe to safely schedule the - asynchronous send operation in the Discord client's event loop, allowing - it to be called from any thread. - - Args: - channel_id (int): The ID of the channel to send the embed message to. - title (str): The title of the embed message. - fields (list): A list of tuples representing the fields of the embed message. - color (str, optional): The color of the embed message. Defaults to 'blue'. - image_url (str, optional): The URL of the image to include in the embed message. - """ - async def send_embed_async(): - try: - channel = self.client.get_channel(channel_id) - if channel: - # Convert color string to discord.Color - embed_color = getattr(discord.Color, color.lower(), discord.Color.blue)() - - embed = discord.Embed( - title=title, - color=embed_color - ) - if image_url: - embed.set_image(url=image_url) - for name, value in fields: - embed.add_field(name=name, value=value, inline=False) - - await channel.send(embed=embed) - else: - self.logger.log(f"Channel {channel_id} not found", 'error', 'DiscordClient') - except discord.errors.Forbidden: - self.logger.log(f"Cannot send embed to channel {channel_id}. Forbidden.", 'error', 'DiscordClient') - except Exception as e: - self.logger.log(f"Error sending embed to channel {channel_id}: {str(e)}", 'error', 'DiscordClient') - - asyncio.run_coroutine_threadsafe(send_embed_async(), self.client.loop) - - def load_commands(self): - """ - Load slash commands for the Discord client. - - This method registers a single slash command ("bot") for the Discord client. - The command is added to the command tree and will be available for users to - interact with in Discord. - """ - name = 'bot' - description = 'send a command to the bot' - function_name = 'bot' - - @discord.app_commands.command(name=name, description=description) - async def command_callback(interaction: discord.Interaction, command: str): - kwargs = {"arg": command} - await self.handle_command(interaction, name, function_name, kwargs) - - param_name = "command" - param_description = "send a command to the bot" - command_callback = discord.app_commands.describe(**{param_name: param_description})(command_callback) - - self.logger.log(f"Register command: {name}, Function: {function_name}", "info", "DiscordClient") - self.tree.add_command(command_callback) - - async def handle_command(self, interaction: discord.Interaction, command_name: str, function_name: str, kwargs: dict): - """ - Handle a slash command interaction. - - This method is called asynchronously by the Discord client when a slash - command is invoked. It adds the command to the message queue for processing. - - Args: - interaction (discord.Interaction): The interaction object for the command. - command_name (str): The name of the command. - function_name (str): The name of the function to handle the command. - kwargs (dict): Additional arguments for the command. - """ - message_data = { - "channel": str(interaction.channel), - "channel_id": interaction.channel_id, - "message": f"/{command_name}", - "author": interaction.user.display_name, - "author_id": interaction.user, - "timestamp": interaction.created_at.strftime('%Y-%m-%d %H:%M:%S'), - "mentions": interaction.data.get("resolved", {}).get("members", []), - "function_name": function_name, - "arg": kwargs.get('arg', None) - } - - if interaction.channel_id not in self.message_queue: - self.message_queue[interaction.channel_id] = [] - self.message_queue[interaction.channel_id].append(message_data) - - await interaction.response.send_message(f"Command '{command_name}' received and added to the queue.") - - async def set_typing_indicator(self, channel_id, is_typing): - """ - Set the typing indicator for a specified Discord channel. - - This method uses asyncio.run_coroutine_threadsafe to safely schedule the - asynchronous typing indicator operation in the Discord client's event loop, - allowing it to be called from any thread. - - Args: - channel_id (int): The ID of the channel to set the typing indicator for. - is_typing (bool): Whether to start or stop the typing indicator. - """ - channel = self.client.get_channel(channel_id) - - if channel: - if is_typing: - async with channel.typing(): - # Keep the typing indicator on for a specific duration - await asyncio.sleep(5) # Adjust the duration as needed - else: - # Stop the typing indicator immediately - await asyncio.sleep(0) - else: - print(f"Channel with ID {channel_id} not found.") - - def add_role(self, guild_id, user_id, role_name): - """ - Add a role to a user in a specified guild. - - This method demonstrates how to run an asynchronous operation synchronously - from an external thread. It uses asyncio.run_coroutine_threadsafe to schedule - the operation in the Discord client's event loop and waits for the result. - - Args: - guild_id (int): The ID of the guild. - user_id (int): The ID of the user. - role_name (str): The name of the role to add. - - Returns: - str: A message indicating the result of the operation. - """ - async def add_role_async(): - try: - guild = self.client.get_guild(guild_id) - if not guild: - return f"Guild with ID {guild_id} not found." - - member = await guild.fetch_member(user_id) - if not member: - return f"User with ID {user_id} not found in the guild." - - role = discord.utils.get(guild.roles, name=role_name) - if not role: - return f"Role '{role_name}' not found in the guild." - - await member.add_roles(role) - return f"Successfully added role '{role_name}' to user {member.name}." - except discord.errors.Forbidden: - return f"Bot doesn't have permission to manage roles." - except Exception as e: - return f"Error adding role: {str(e)}" - - return asyncio.run_coroutine_threadsafe(add_role_async(), self.client.loop).result() - - def remove_role(self, guild_id, user_id, role_name): - """ - Remove a role from a user in a specified guild. - - This method demonstrates how to run an asynchronous operation synchronously - from an external thread. It uses asyncio.run_coroutine_threadsafe to schedule - the operation in the Discord client's event loop and waits for the result. - - Args: - guild_id (int): The ID of the guild. - user_id (int): The ID of the user. - role_name (str): The name of the role to remove. - - Returns: - str: A message indicating the result of the operation. - """ - async def remove_role_async(): - try: - guild = self.client.get_guild(guild_id) - if not guild: - return f"Guild with ID {guild_id} not found." - - member = await guild.fetch_member(user_id) - if not member: - return f"User with ID {user_id} not found in the guild." - - role = discord.utils.get(guild.roles, name=role_name) - if not role: - return f"Role '{role_name}' not found in the guild." - - await member.remove_roles(role) - return f"Successfully removed role '{role_name}' from user {member.name}." - except discord.errors.Forbidden: - return f"Bot doesn't have permission to manage roles." - except Exception as e: - return f"Error removing role: {str(e)}" - - return asyncio.run_coroutine_threadsafe(remove_role_async(), self.client.loop).result() - - def has_role(self, guild_id, user_id, role_name): - """ - Check if a user has a specified role in a guild. - - This method demonstrates how to run an asynchronous operation synchronously - from an external thread. It uses asyncio.run_coroutine_threadsafe to schedule - the operation in the Discord client's event loop and waits for the result. - - Args: - guild_id (int): The ID of the guild. - user_id (int): The ID of the user. - role_name (str): The name of the role to check. - - Returns: - bool: True if the user has the role, False otherwise. - """ - async def has_role_async(): - try: - guild = self.client.get_guild(guild_id) - if not guild: - return f"Guild with ID {guild_id} not found." - - member = await guild.fetch_member(user_id) - if not member: - return f"User with ID {user_id} not found in the guild." - - role = discord.utils.get(guild.roles, name=role_name) - if not role: - return f"Role '{role_name}' not found in the guild." - - return role in member.roles - except Exception as e: - return f"Error checking role: {str(e)}" - - return asyncio.run_coroutine_threadsafe(has_role_async(), self.client.loop).result() - - def list_roles(self, guild_id, user_id=None): - """ - List roles in a guild and optionally for a specific user. - - This method demonstrates how to run an asynchronous operation synchronously - from an external thread. It uses asyncio.run_coroutine_threadsafe to schedule - the operation in the Discord client's event loop and waits for the result. - - Args: - guild_id (int): The ID of the guild. - user_id (int, optional): The ID of the user. If provided, the method will also list the user's roles. - - Returns: - str: A formatted string listing the roles in the guild and optionally for the user. - """ - async def list_roles_async(): - try: - guild = self.client.get_guild(guild_id) - if not guild: - return f"Guild with ID {guild_id} not found." - - # List all guild roles - all_roles = [f"{role.name} (ID: {role.id})" for role in guild.roles if role.name != "@everyone"] - guild_roles = "Guild Roles:\n" + "\n".join(all_roles) if all_roles else "No roles found in this guild." - - # List user roles if user_id is provided - user_roles = "" - if user_id: - member = await guild.fetch_member(user_id) - if member: - user_role_list = [f"{role.name} (ID: {role.id})" for role in member.roles if - role.name != "@everyone"] - user_roles = f"\n\nRoles for user {member.name}:\n" + "\n".join( - user_role_list) if user_role_list else f"\n\nUser {member.name} has no roles." - else: - user_roles = f"\n\nUser with ID {user_id} not found in the guild." - - return guild_roles + user_roles - except Exception as e: - return f"Error listing roles: {str(e)}" - - return asyncio.run_coroutine_threadsafe(list_roles_async(), self.client.loop).result() - - def create_thread(self, channel_id, message_id, name, auto_archive_duration=1440, remove_author=True): - """ - Create a new thread in a specified channel, attached to a specific message. - - This method uses asyncio.run_coroutine_threadsafe to safely schedule the - asynchronous thread creation operation in the Discord client's event loop, - allowing it to be called from any thread. - - Args: - channel_id (int): The ID of the channel to create the thread in. - message_id (int): The ID of the message to attach the thread to. - name (str): The name of the new thread. - auto_archive_duration (int, optional): Duration in minutes after which the thread - will automatically archive. Default is 1440 (24 hours). - remove_author (bool, optional): Whether to remove the message author from the thread. Default is False. - - Returns: - int: The ID of the created thread, or None if creation failed. - """ - async def create_thread_async(): - try: - channel = self.client.get_channel(channel_id) - if not channel: - self.logger.log(f"Channel {channel_id} not found", 'error', 'DiscordClient') - return None - - message = await channel.fetch_message(message_id) - if not message: - self.logger.log(f"Message {message_id} not found in channel {channel_id}", 'error', 'DiscordClient') - return None - - thread = await message.create_thread(name=name, auto_archive_duration=auto_archive_duration) - self.logger.log(f"Thread '{name}' created successfully", 'info', 'DiscordClient') - - if remove_author: - await thread.remove_user(message.author) - self.logger.log(f"Removed author {message.author} from thread '{name}'", 'info', 'DiscordClient') - - return thread.id - except discord.errors.Forbidden: - self.logger.log(f"Bot doesn't have permission to create threads in channel {channel_id}", 'error', 'DiscordClient') - except Exception as e: - self.logger.log(f"Error creating thread: {str(e)}", 'error', 'DiscordClient') - return None - - return asyncio.run_coroutine_threadsafe(create_thread_async(), self.client.loop).result() - - def reply_to_thread(self, thread_id, content): - """ - Reply to a specific thread. - - This method uses asyncio.run_coroutine_threadsafe to safely schedule the - asynchronous reply operation in the Discord client's event loop, - allowing it to be called from any thread. - - Args: - thread_id (int): The ID of the thread to reply to. - content (str): The content of the reply message. - - Returns: - bool: True if the reply was sent successfully, False otherwise. - """ - async def reply_async(): - try: - thread = self.client.get_channel(thread_id) - if not thread: - self.logger.log(f"Thread {thread_id} not found", 'error', 'DiscordClient') - return False - - # Split the content into semantic chunks - chunks = semantic_chunk(content, min_length=200, max_length=1900) - for i, chunk in enumerate(chunks, 1): - message = f"```chunk.content```" - await thread.send(message) - - self.logger.log(f"Reply sent to thread {thread_id}", 'info', 'DiscordClient') - return True - except discord.errors.Forbidden: - self.logger.log(f"Bot doesn't have permission to reply to thread {thread_id}", 'error', 'DiscordClient') - except Exception as e: - self.logger.log(f"Error replying to thread: {str(e)}", 'error', 'DiscordClient') - return False - - return asyncio.run_coroutine_threadsafe(reply_async(), self.client.loop).result() - - -if __name__ == "__main__": - # This is just for testing the DiscordClient - import time - - client = DiscordClient() - client.run() - - while True: - try: - for message_item in client.process_channel_messages(): - time.sleep(5) - client.send_message(message_item[0], f"Processed: {message_item}") - except KeyboardInterrupt: - print("Stopping...") - client.stop() - finally: - time.sleep(5) diff --git a/src/agentforge/utils/Logger.py b/src/agentforge/utils/Logger.py deleted file mode 100755 index 82942ed5..00000000 --- a/src/agentforge/utils/Logger.py +++ /dev/null @@ -1,317 +0,0 @@ -import os -import logging -from agentforge.config import Config - -from termcolor import cprint -from colorama import init -init(autoreset=True) - - -def encode_msg(msg): - return msg.encode('utf-8', 'replace').decode('utf-8') - - -class BaseLogger: - """ - A base logger class for setting up file and console logging with support for multiple handlers and log levels. - - This class provides mechanisms for initializing file and console log handlers, logging messages at various - levels, and dynamically adjusting log levels. - - Attributes: - file_handlers (dict): A class-level dictionary tracking file handlers by log file name. - console_handlers (dict): A class-level dictionary tracking console handlers by logger name. - """ - - # Class-level dictionaries to track existing handlers - file_handlers = {} - console_handlers = {} - - def __init__(self, name='BaseLogger', log_file='default.log', log_level='error'): - """ - Initializes the BaseLogger with optional name, log file, and log level. - - Parameters: - name (str): The name of the logger. - log_file (str): The name of the file to log messages to. - log_level (str): The initial log level for the logger. - """ - self.config = Config() - - # Retrieve the logging enabled flag from configuration - logging_enabled = self.config.data['settings']['system']['Logging']['Enabled'] - - level = self._get_level_code(log_level) - self.logger = logging.getLogger(name) - self.log_folder = None - self.log_file = log_file - - # Conditional setup based on logging enabled flag - if logging_enabled: - self._setup_file_handler(level) - self._setup_console_handler(level) - self.logger.setLevel(level) - return - - # If logging is disabled, set the logger level to NOTSET or higher than CRITICAL to effectively disable it - self.logger.setLevel(logging.CRITICAL + 1) # Effectively disables logging - - @staticmethod - def _get_level_code(level): - """ - Converts a log level as a string to the corresponding logging module level code. - - Parameters: - level (str): The log level as a string (e.g., 'debug', 'info', 'warning', 'error', 'critical'). - - Returns: - int: The logging module level code corresponding to the provided string. - """ - level_dict = { - 'debug': logging.DEBUG, - 'info': logging.INFO, - 'warning': logging.WARNING, - 'error': logging.ERROR, - 'critical': logging.CRITICAL, - } - return level_dict.get(level.lower(), logging.INFO) - - def _setup_console_handler(self, level): - """ - Sets up a console handler for logging messages to the console. Configures logging format and level. - - Parameters: - level (int): The logging level to set for the console handler. - """ - - formatter = logging.Formatter(f'%(asctime)s - %(levelname)s - {self.log_file} - %(message)s\n', - datefmt='%Y-%m-%d %H:%M:%S') - - if self.logger.name in BaseLogger.console_handlers: - # Use the existing console handler if it's not already added to this logger - ch = BaseLogger.console_handlers[self.logger.name] - if ch not in self.logger.handlers: - ch.setLevel(level) - ch.setFormatter(formatter) - self.logger.addHandler(ch) - return - - # Console handler for logs - ch = logging.StreamHandler() - ch.setLevel(level) - ch.setFormatter(formatter) - - if not any(type(handler) is logging.StreamHandler for handler in self.logger.handlers): - self.logger.addHandler(ch) - - BaseLogger.console_handlers[self.logger.name] = ch - - def _setup_file_handler(self, level): - """ - Sets up a file handler for logging messages to a file. Initializes the log folder and file if they do not exist, - and configures logging format and level. - - Parameters: - level (int): The logging level to set for the file handler. - """ - # Create the Logs folder if it doesn't exist - self.initialize_logging() - - formatter = logging.Formatter(f'%(asctime)s - %(levelname)s - %(message)s\n' - '-------------------------------------------------------------', - datefmt='%Y-%m-%d %H:%M:%S') - - if self.log_file in BaseLogger.file_handlers: - # Use the existing file handler if it's not already added to this logger - fh = BaseLogger.file_handlers[self.log_file] - if fh not in self.logger.handlers: - fh.setLevel(level) - fh.setFormatter(formatter) - self.logger.addHandler(fh) - return - - # File handler for logs - log_file_path = f'{self.log_folder}/{self.log_file}' - fh = logging.FileHandler(log_file_path, encoding='utf-8') - fh.setLevel(level) - fh.setFormatter(formatter) - - # Check if a similar handler is already attached - if not any(isinstance(handler, logging.FileHandler) and handler.baseFilename == fh.baseFilename for handler in - self.logger.handlers): - self.logger.addHandler(fh) - - # Store the file handler in the class-level dictionary - BaseLogger.file_handlers[self.log_file] = fh - - def initialize_logging(self): - """ - Initializes logging by ensuring the log folder exists and setting up the log folder path. - """ - # Save the result to a log.txt file in the /Logs/ folder - self.log_folder = self.config.data['settings']['system']['Logging']['Folder'] - self.logger.handlers = [h for h in self.logger.handlers if not isinstance(h, logging.StreamHandler)] - - # Create the Logs folder if it doesn't exist - if not os.path.exists(self.log_folder): - os.makedirs(self.log_folder) - return - - def log_msg(self, msg, level='info'): - """ - Logs a message at the specified log level. - - Parameters: - msg (str): The message to log. - level (str): The level at which to log the message (e.g., 'info', 'debug', 'error'). - """ - level_code = self._get_level_code(level) - - if level_code == logging.DEBUG: - self.logger.debug(msg) - elif level_code == logging.INFO: - self.logger.info(msg) - elif level_code == logging.WARNING: - self.logger.warning(msg) - elif level_code == logging.ERROR: - self.logger.error(msg) - self.logger.exception("Exception Error Occurred!") - elif level_code == logging.CRITICAL: - self.logger.critical(msg) - self.logger.exception("Critical Exception Occurred!") - raise - else: - raise ValueError(f'Invalid log level: {level}') - - def set_level(self, level): - """ - Sets the log level for the logger and its handlers. - - Parameters: - level (str): The new log level to set (e.g., 'info', 'debug', 'error'). - """ - level_code = self._get_level_code(level) - self.logger.setLevel(level_code) - for handler in self.logger.handlers: - handler.setLevel(level_code) - - -class Logger: - _instances = {} - """ - A wrapper class for managing multiple BaseLogger instances, supporting different log files and levels - as configured in the system settings. - - This class facilitates logging across different modules and components of the application, allowing - for specific logs for agent activities, model interactions, and results. - - Attributes: - loggers (dict): A dictionary of BaseLogger instances keyed by log type (e.g., '.agentforge', 'modelio'). - """ - - def __new__(cls, name: str): - """ - Create a new instance of Logger if one doesn't exist, or return the existing instance. - - Parameters: - name (str): The name of the module or component using the logger. - """ - if name not in cls._instances: - instance = super(Logger, cls).__new__(cls) - cls._instances[name] = instance - instance._initialized = False - return cls._instances[name] - - def __init__(self, name: str): - """ - Initializes the Logger class with names for different types of logs. Initialization will only happen once. - - Parameters: - name (str): The name of the module or component using the logger. - """ - if self._initialized: - return - - self.config = Config() - self.caller_name = name # This will store the __name__ of the script that instantiated the Logger - - # Retrieve the logging configuration from the config data - logging_config = self.config.data['settings']['system']['Logging']['Files'] - - # Initialize loggers dynamically based on configuration settings - self.loggers = {} - for log_name, log_level in logging_config.items(): - log_file_name = f'{log_name}.log' - new_logger = BaseLogger(name=f'{name}.{log_name}', log_file=log_file_name, log_level=log_level) - self.loggers[log_name] = new_logger - - self._initialized = True - - def log(self, msg: str, level: str = 'info', logger_file: str = 'AgentForge'): - """ - Logs a message to a specified logger or all loggers. - - Parameters: - msg (str): The message to log. - level (str): The log level (e.g., 'info', 'debug', 'error'). - logger_file (str): The specific logger to use, or 'all' to log to all loggers. - """ - # Prepend the caller's module name to the log message - msg_with_caller = f'[{self.caller_name}]\n{msg}' - - if logger_file not in self.loggers: - raise ValueError(f"Unknown logger file '{logger_file}' - Make sure the file name is a Logging File in " - f"the configuration file (system.yaml).") - - self.loggers[logger_file].log_msg(msg_with_caller, level) - - def log_prompt(self, model_prompt: dict[str]): - """ - Logs a prompt to the model interaction logger. - - Parameters: - model_prompt (dict[str]): A dictionary containing the model prompts for generating a completion. - """ - system_prompt = model_prompt.get('System') - user_prompt = model_prompt.get('User') - msg = ( - f'******\nSystem Prompt\n******\n{system_prompt}\n' - f'******\nUser Prompt\n******\n{user_prompt}\n' - f'******' - ) - self.log(msg, 'debug', 'ModelIO') - - def log_response(self, response: str): - """ - Logs a model response to the model interaction logger. - - Parameters: - response (str): The model response to log. - """ - msg = f'******\nModel Response\n******\n{response}\n******' - self.log(msg, 'debug', 'ModelIO') - - def parsing_error(self, model_response: str, error: Exception): - """ - Logs parsing errors along with the model response. - - Parameters: - model_response (str): The model response associated with the parsing error. - error (Exception): The exception object representing the parsing error. - """ - self.log(f"Parsing Error - It is very likely the model did not respond in the required " - f"format\n\nModel Response\n******\n{model_response}\n******\n\nError: {error}", 'error') - - def log_info(self, msg: str): - """ - Logs and displays an informational message. - - Parameters: - msg (str): The message to log and display. - """ - try: - encoded_msg = encode_msg(msg) # Utilize the existing encode_msg function - cprint(encoded_msg, 'red', attrs=['bold']) - self.log(f'\n{encoded_msg}', 'info', 'Results') - except Exception as e: - self.log(f"Error logging message: {e}", 'error') diff --git a/src/agentforge/utils/ParsingUtils.py b/src/agentforge/utils/ParsingUtils.py deleted file mode 100644 index 13c30381..00000000 --- a/src/agentforge/utils/ParsingUtils.py +++ /dev/null @@ -1,64 +0,0 @@ -import re -import yaml -from typing import Optional, Dict, Any -from agentforge.utils.Logger import Logger - - -class ParsingUtils: - - def __init__(self): - """ - Initializes the ParsingUtils class with a Logger instance. - """ - self.logger = Logger(name=self.__class__.__name__) - - def extract_yaml_block(self, text: str) -> Optional[str]: - """ - Extracts a YAML block from a string, typically used to parse YAML content from larger text blocks or files. - If no specific YAML block is found, attempts to extract from a generic code block or returns the entire text. - - Parameters: - text (str): The text containing the YAML block. - - Returns: - Optional[str]: The extracted YAML block as a string, or None if no valid YAML content is found. - """ - try: - # Regex pattern to capture content between ```yaml and ``` - yaml_pattern = r"```yaml(.*?)```" - match = re.search(yaml_pattern, text, re.DOTALL) - - if match: - return match.group(1).strip() - - # If no specific YAML block is found, try to capture content between generic code block ``` - code_block_pattern = r"```(.*?)```" - match = re.search(code_block_pattern, text, re.DOTALL) - - if match: - return match.group(1).strip() - - # If no code block is found, return the entire text - return text.strip() - except Exception as e: - self.logger.log(f"Regex Error Extracting YAML Block: {e}", 'error') - return None - - def parse_yaml_content(self, yaml_string: str) -> Optional[Dict[str, Any]]: - """ - Parses a YAML-formatted string into a Python dictionary. - - Parameters: - yaml_string (str): The YAML string to parse. - - Returns: - Optional[Dict[str, Any]]: The parsed YAML content as a dictionary, or None if parsing fails. - """ - try: - cleaned_string = self.extract_yaml_block(yaml_string) - if cleaned_string: - return yaml.safe_load(cleaned_string) - return None - except yaml.YAMLError as e: - self.logger.parsing_error(yaml_string, e) - return None diff --git a/src/agentforge/utils/discord/discord_client.py b/src/agentforge/utils/discord/discord_client.py new file mode 100644 index 00000000..516f23f4 --- /dev/null +++ b/src/agentforge/utils/discord/discord_client.py @@ -0,0 +1,323 @@ +# utils/discord_client.py + +import discord +import os +import asyncio +import threading +from agentforge.utils.logger import Logger +from agentforge.tools.semantic_chunk import semantic_chunk +from agentforge.utils.discord.discord_utils import DiscordUtils + + + +class DiscordClient: + """ + A Discord client that handles bot functionality, message processing, and role management. + + This class uses a combination of asyncio and threading to manage Discord operations: + - The Discord client runs in a separate thread to avoid blocking the main application. + - Asynchronous methods are used for Discord API calls, which are then run in the client's event loop. + - Thread-safe methods are provided for external code to interact with the Discord client. + + Attributes: + token (str): The Discord bot token. + intents (discord.Intents): The intents for the Discord client. + client (discord.Client): The main Discord client instance. + logger (Logger): A custom logger for the Discord client. + tree (discord.app_commands.CommandTree): The command tree for slash commands. + message_queue (dict): A queue to store incoming messages, keyed by channel ID. + running (bool): A flag indicating whether the client is running. + discord_thread (threading.Thread): The thread running the Discord client. + """ + + def __init__(self): + """ + Initialize the DiscordClient with necessary attributes and event handlers. + """ + self.discord_thread = None + self.token = str(os.getenv('DISCORD_TOKEN')) + self.intents = discord.Intents.default() + self.intents.message_content = True + self.client = discord.Client(intents=self.intents) + self.logger = Logger('DiscordClient', 'DiscordClient') + self.tree = discord.app_commands.CommandTree(self.client) + self.message_queue = {} + self.running = False + self.load_commands() + self.utils = DiscordUtils(self.client, self.logger) + + + @self.client.event + async def on_ready(): + await self.tree.sync() + self.logger.info(f'[DiscordClient.on_ready] {self.client.user} has connected to Discord!') + + @self.client.event + async def on_message(message: discord.Message): + self.logger.debug(f"[DiscordClient.on_message] Received message:\n{message}") + + content = message.content + for mention in message.mentions: + # If a mention is copy/pasted, this does not work. The mention value will come through as Null. + content = content.replace(f'<@{mention.id}>', f'@{mention.display_name}') + + message_data = { + "channel": str(message.channel), + "channel_id": message.channel.id, + "message": content, + "message_id": message.id, + "author": message.author.display_name, + "author_id": message.author, + "timestamp": message.created_at.strftime('%Y-%m-%d %H:%M:%S'), + "mentions": message.mentions, + "attachments": message.attachments + } + + # Add thread information to message_data if the message is in a thread + if isinstance(message.channel, discord.Thread): + message_data["thread_id"] = message.channel.id + message_data["thread_name"] = message.channel.name + + self.logger.debug( + f"[DiscordClient.on_message] Channel: {str(message.channel)}({message.channel.id}) - {message.author.display_name} said:\n{content}") + + if message.author != self.client.user: + if message.channel.id not in self.message_queue: + self.message_queue[message.channel.id] = [] + self.message_queue[message.channel.id].append(message_data) + self.logger.debug("[DiscordClient.on_message] Message added to queue") + else: + self.logger.debug(f"[DiscordClient.on_message] Message not added to queue:\n{message_data}") + + def run(self): + """ + Start the Discord client in a separate thread. + + This method creates a new thread that runs the Discord client's event loop. + The thread allows the Discord client to operate independently of the main + application thread, preventing it from blocking other operations. + """ + + def run_discord(): + self.logger.info("[DiscordClient.run] Client Starting") + asyncio.run(self.client.start(self.token)) + + self.discord_thread = threading.Thread(target=run_discord) + self.discord_thread.start() + self.running = True + + def stop(self): + """ + Stop the Discord client and join the client thread. + + This method closes the Discord client's connection and waits for the + client thread to finish, ensuring a clean shutdown. + """ + self.running = False + asyncio.run(self.client.close()) + self.discord_thread.join() + self.logger.info("[DiscordClient.stop] Client Stopped") + + def process_channel_messages(self): + """ + Process and yield messages from the message queue. + + This function retrieves all messages sent to a discord channel from the + message_queue and yields them. + Each message is represented as a tuple with the following structure: + + (channel_id, [message_data]) + + where: + - channel_id (int): The ID of the Discord channel where the message was sent. + - message_data (list): A list containing a single dictionary with message details: + { + 'channel': str, # The name of the channel (e.g., 'system') + 'channel_id': int, # The ID of the channel (same as the tuple's first element) + 'message': str, # The content of the message + 'author': str, # The display name of the message author + 'author_id': Member, # The Discord Member object of the author + 'timestamp': str # The timestamp of the message in 'YYYY-MM-DD HH:MM:SS' format + 'mentions': list # A list of Discord Member objects mentioned in the message + } + + Yields: + tuple: A message tuple as described above. + + Note: + - This function is designed to work with Discord message objects. + - If the message queue is empty, the function will print "No Message Found" and pass. + - Any exceptions during message processing will be caught and printed. + """ + if self.message_queue: + try: + next_message = self.message_queue.popitem() + yield next_message + except Exception as e: + print(f"Exception: {e}") + else: + pass + + def send_message(self, channel_id, content): + """ + Send a message to a specified Discord channel. + + Args: + channel_id (int): The ID of the channel to send the message to. + content (str): The content of the message to send. + """ + self.utils.send_message(channel_id, content) + + def send_dm(self, user_id, content): + """ + Send a direct message to a specified Discord user. + + Args: + user_id (int): The ID of the user to send the direct message to. + content (str): The content of the direct message to send. + """ + self.utils.send_dm(user_id, content) + + def send_embed(self, channel_id, title, fields, color='blue', image_url=None): + """ + Send an embed message to a specified Discord channel. + + Args: + channel_id (int): The ID of the channel to send the embed message to. + title (str): The title of the embed message. + fields (list): A list of tuples representing the fields of the embed message. + color (str, optional): The color of the embed message. Defaults to 'blue'. + image_url (str, optional): The URL of the image to include in the embed message. + """ + self.utils.send_embed(channel_id, title, fields, color, image_url) + + def load_commands(self): + """ + Load slash commands for the Discord client. + + This method registers a single slash command ("bot") for the Discord client. + The command is added to the command tree and will be available for users to + interact with in Discord. + """ + name = 'bot' + description = 'send a command to the bot' + function_name = 'bot' + + @discord.app_commands.command(name=name, description=description) + async def command_callback(interaction: discord.Interaction, command: str): + kwargs = {"arg": command} + await self.handle_command(interaction, name, function_name, kwargs) + + param_name = "command" + param_description = "send a command to the bot" + command_callback = discord.app_commands.describe(**{param_name: param_description})(command_callback) + + self.logger.info(f"[DiscordClient.load_commands] Register Command: {name} - Function: {function_name}") + self.tree.add_command(command_callback) + + async def handle_command(self, interaction: discord.Interaction, command_name: str, function_name: str, + kwargs: dict): + """ + Handle a slash command interaction. + + This method is called asynchronously by the Discord client when a slash + command is invoked. It adds the command to the message queue for processing. + + Args: + interaction (discord.Interaction): The interaction object for the command. + command_name (str): The name of the command. + function_name (str): The name of the function to handle the command. + kwargs (dict): Additional arguments for the command. + """ + message_data = { + "channel": str(interaction.channel), + "channel_id": interaction.channel_id, + "message": f"/{command_name}", + "author": interaction.user.display_name, + "author_id": interaction.user, + "timestamp": interaction.created_at.strftime('%Y-%m-%d %H:%M:%S'), + "mentions": interaction.data.get("resolved", {}).get("members", []), + "function_name": function_name, + "arg": kwargs.get('arg', None) + } + + if interaction.channel_id not in self.message_queue: + self.message_queue[interaction.channel_id] = [] + self.message_queue[interaction.channel_id].append(message_data) + self.logger.info(f"[DiscordClient.handle_command] Command '{command_name}' received and added to the queue") + + await interaction.response.send_message(f"Command '{command_name}' received and added to the queue.") + + async def set_typing_indicator(self, channel_id, is_typing): + """ + Set the typing indicator for a specified Discord channel. + + This method uses asyncio.run_coroutine_threadsafe to safely schedule the + asynchronous typing indicator operation in the Discord client's event loop, + allowing it to be called from any thread. + + Args: + channel_id (int): The ID of the channel to set the typing indicator for. + is_typing (bool): Whether to start or stop the typing indicator. + """ + channel = self.client.get_channel(channel_id) + + if channel: + if is_typing: + async with channel.typing(): + # Keep the typing indicator on for a specific duration + await asyncio.sleep(5) # Adjust the duration as needed + else: + # Stop the typing indicator immediately + await asyncio.sleep(0) + else: + self.logger.error(f"Channel with ID {channel_id} not found.") + + def create_thread(self, channel_id, message_id, name, auto_archive_duration=1440, remove_author=True): + """ + Create a new thread in a specified channel, attached to a specific message. + + Args: + channel_id (int): The ID of the channel to create the thread in. + message_id (int): The ID of the message to attach the thread to. + name (str): The name of the new thread. + auto_archive_duration (int, optional): Duration in minutes after which the thread + will automatically archive. Default is 1440 (24 hours). + remove_author (bool, optional): Whether to remove the message author from the thread. Default is True. + + Returns: + int: The ID of the created thread, or None if creation failed. + """ + return self.utils.create_thread(channel_id, message_id, name, auto_archive_duration, remove_author) + + def reply_to_thread(self, thread_id, content): + """ + Reply to a specific thread. + + Args: + thread_id (int): The ID of the thread to reply to. + content (str): The content of the reply message. + + Returns: + bool: True if the reply was sent successfully, False otherwise. + """ + return self.utils.reply_to_thread(thread_id, content) + + +if __name__ == "__main__": + # This is just for testing the DiscordClient + import time + + client = DiscordClient() + client.run() + + while True: + try: + for message_item in client.process_channel_messages(): + time.sleep(5) + client.send_message(message_item[0], f"Processed: {message_item}") + except KeyboardInterrupt: + print("Stopping...") + client.stop() + finally: + time.sleep(5) diff --git a/src/agentforge/utils/discord/discord_utils.py b/src/agentforge/utils/discord/discord_utils.py new file mode 100644 index 00000000..a1968e82 --- /dev/null +++ b/src/agentforge/utils/discord/discord_utils.py @@ -0,0 +1,223 @@ +import asyncio +import discord +from agentforge.tools.semantic_chunk import semantic_chunk + +class DiscordUtils: + def __init__(self, client, logger): + """ + Initialize Discord utilities with client and logger instances. + + Args: + client (discord.Client): The Discord client instance + logger (Logger): Logger instance for error handling + """ + self.client = client + self.logger = logger + + def send_message(self, channel_id, content): + """ + Send a message to a specified Discord channel. + + Args: + channel_id (int): The ID of the channel to send the message to + content (str): The content of the message to send + """ + async def send(): + try: + messages = semantic_chunk(content, min_length=200, max_length=1900) + channel = self.client.get_channel(channel_id) + + if not channel: + self.logger.error(f"[DiscordUtils.send_message] Channel {channel_id} not found") + return + + for msg in messages: + if len(msg.content) > 2000: + # Re-chunk the over-sized message + sub_messages = semantic_chunk(msg.content, min_length=200, max_length=1900) + for sub_msg in sub_messages: + await channel.send(sub_msg.content) + else: + await channel.send(msg.content) + + except discord.errors.Forbidden: + self.logger.error(f"[DiscordUtils.send_message] Bot doesn't have permission to send messages in channel {channel_id}") + except Exception as e: + self.logger.error(f"[DiscordUtils.send_message] Error sending message to channel {channel_id}: {str(e)}") + + try: + asyncio.run_coroutine_threadsafe(send(), self.client.loop) + except RuntimeError as e: + self.logger.error(f"[DiscordUtils.send_message] Failed to schedule message sending: {str(e)}") + + def send_dm(self, user_id, content): + """ + Send a direct message to a specified Discord user. + + Args: + user_id (int): The ID of the user to send the direct message to + content (str): The content of the direct message to send + """ + async def send_dm_async(): + try: + user = await self.client.fetch_user(user_id) + if user: + messages = semantic_chunk(content, min_length=200, max_length=1900) + for msg in messages: + if len(msg.content) > 2000: + # Re-chunk the over-sized message + sub_messages = semantic_chunk(msg.content, min_length=200, max_length=1900) + for sub_msg in sub_messages: + await user.send(sub_msg.content) + else: + await user.send(msg.content) + else: + self.logger.error(f"[DiscordUtils.send_dm] User {user_id} not found") + except discord.errors.NotFound: + self.logger.error(f"[DiscordUtils.send_dm] User {user_id} not found") + except discord.errors.Forbidden: + self.logger.error(f"[DiscordUtils.send_dm] Cannot send DM to user {user_id}. Forbidden.") + except Exception as e: + self.logger.error(f"[DiscordUtils.send_dm] Error sending DM to user {user_id}: {str(e)}") + + try: + asyncio.run_coroutine_threadsafe(send_dm_async(), self.client.loop) + except RuntimeError as e: + self.logger.error(f"[DiscordUtils.send_dm] Failed to schedule DM sending: {str(e)}") + + def send_embed(self, channel_id, title, fields, color='blue', image_url=None): + """ + Send an embed message to a specified Discord channel. + + Args: + channel_id (int): The ID of the channel to send the embed message to + title (str): The title of the embed message + fields (list): A list of tuples representing the fields of the embed message + color (str, optional): The color of the embed message. Defaults to 'blue' + image_url (str, optional): The URL of the image to include in the embed message + """ + async def send_embed_async(): + try: + channel = self.client.get_channel(channel_id) + if channel: + # Convert color string to discord.Color + embed_color = getattr(discord.Color, color.lower(), discord.Color.blue)() + + embed = discord.Embed( + title=title, + color=embed_color + ) + if image_url: + embed.set_image(url=image_url) + for name, value in fields: + embed.add_field(name=name, value=value, inline=False) + + await channel.send(embed=embed) + else: + self.logger.error(f"[DiscordUtils.send_embed] Channel with ID {channel_id} not found") + except discord.errors.Forbidden: + self.logger.error(f"[DiscordUtils.send_embed] Cannot send embed to channel {channel_id}. Forbidden") + except Exception as e: + self.logger.error(f"[DiscordUtils.send_embed] Error sending embed to channel {channel_id}: {str(e)}") + + try: + asyncio.run_coroutine_threadsafe(send_embed_async(), self.client.loop) + except RuntimeError as e: + self.logger.error(f"[DiscordUtils.send_embed] Failed to schedule embed sending: {str(e)}") + + def create_thread(self, channel_id, message_id, name, auto_archive_duration=1440, remove_author=True): + """ + Create a new thread in a specified channel, attached to a specific message. + + Args: + channel_id (int): The ID of the channel to create the thread in + message_id (int): The ID of the message to attach the thread to + name (str): The name of the new thread + auto_archive_duration (int, optional): Duration in minutes after which the thread + will automatically archive. Default is 1440 (24 hours) + remove_author (bool, optional): Whether to remove the message author from the thread. Default is True + + Returns: + int: The ID of the created thread, or None if creation failed + """ + async def create_thread_async(): + try: + channel = self.client.get_channel(channel_id) + if not channel: + self.logger.error(f"[DiscordUtils.create_thread] Channel with ID {channel_id} not found") + return None + + message = await channel.fetch_message(message_id) + if not message: + self.logger.error(f"[DiscordUtils.create_thread] Message with ID {message_id} not found in channel {channel_id}") + return None + + # Safely check if thread exists using hasattr + if hasattr(message, 'thread') and message.thread: + self.logger.info(f"[DiscordUtils.create_thread] Thread already exists for message {message_id}") + return message.thread.id + + thread = await message.create_thread(name=name, auto_archive_duration=auto_archive_duration) + self.logger.info(f"[DiscordUtils.create_thread] Thread '{name}' created successfully") + + if remove_author: + await thread.remove_user(message.author) + self.logger.info(f"[DiscordUtils.create_thread] Removed author {message.author} from thread '{name}'") + + return thread.id + except discord.errors.HTTPException as e: + if e.code == 160004: # Thread already exists error code + if message.thread: + return message.thread.id + self.logger.error(f"[DiscordUtils.create_thread] Thread exists but cannot be accessed") + else: + self.logger.error(f"[DiscordUtils.create_thread] Error creating thread: {str(e)}") + except discord.errors.Forbidden: + self.logger.error(f"[DiscordUtils.create_thread] Bot doesn't have permission to create threads in channel {channel_id}") + except Exception as e: + self.logger.error(f"[DiscordUtils.create_thread] Error creating thread: {str(e)}") + return None + + try: + return asyncio.run_coroutine_threadsafe(create_thread_async(), self.client.loop).result() + except RuntimeError as e: + self.logger.error(f"[DiscordUtils.create_thread] Failed to schedule thread creation: {str(e)}") + return None + + def reply_to_thread(self, thread_id, content): + """ + Reply to a specific thread. + + Args: + thread_id (int): The ID of the thread to reply to + content (str): The content of the reply message + + Returns: + bool: True if the reply was sent successfully, False otherwise + """ + async def reply_async(): + try: + thread = self.client.get_channel(thread_id) + if not thread: + self.logger.error(f"[DiscordUtils.reply_to_thread] Thread {thread_id} not found") + return False + + # Split the content into semantic chunks + chunks = semantic_chunk(content, min_length=200, max_length=1900) + for i, chunk in enumerate(chunks, 1): + message = f"```{chunk.content}```" + await thread.send(message) + + self.logger.info(f"[DiscordUtils.reply_to_thread] Reply sent to thread {thread_id}") + return True + except discord.errors.Forbidden: + self.logger.error(f"[DiscordUtils.reply_to_thread] Bot doesn't have permission to reply to thread {thread_id}") + except Exception as e: + self.logger.error(f"[DiscordUtils.reply_to_thread] Error replying to thread: {str(e)}") + return False + + try: + return asyncio.run_coroutine_threadsafe(reply_async(), self.client.loop).result() + except RuntimeError as e: + self.logger.error(f"[DiscordUtils.reply_to_thread] Failed to schedule reply: {str(e)}") + return False diff --git a/src/agentforge/utils/logger.py b/src/agentforge/utils/logger.py new file mode 100755 index 00000000..c8c7f17a --- /dev/null +++ b/src/agentforge/utils/logger.py @@ -0,0 +1,356 @@ +import os +import re +import logging +import threading +from agentforge.config import Config + + +def encode_msg(msg: str) -> str: + """Encodes a message to UTF-8, replacing any invalid characters.""" + return msg.encode('utf-8', 'replace').decode('utf-8') + + +class ColoredFormatter(logging.Formatter): + """ + A custom logging formatter to add colors to console logs based on the log level. + Uses ANSI escape codes for coloring without external dependencies. + """ + + COLOR_CODES = { + logging.DEBUG: '\033[36m', # Cyan + logging.INFO: '\033[32m', # Green + logging.WARNING: '\033[33m', # Yellow + logging.ERROR: '\033[31m', # Red + logging.CRITICAL: '\033[41m', # Red background + } + RESET_CODE = '\033[0m' + + def format(self, record: logging.LogRecord) -> str: + color_code = self.COLOR_CODES.get(record.levelno, self.RESET_CODE) + message = super().format(record) + return f"{color_code}{message}{self.RESET_CODE}" + + +class BaseLogger: + """ + A base logger class for setting up file and console logging with support for multiple handlers and log levels. + + This class provides mechanisms for initializing file and console log handlers, logging messages at various + levels, and dynamically adjusting log levels. + + Attributes: + file_handlers (dict): A class-level dictionary tracking file handlers by log file name. + console_handlers (dict): A class-level dictionary tracking console handlers by logger name. + """ + + # Class-level dictionaries to track existing handlers + file_handlers = {} + console_handlers = {} + + def __init__(self, name: str = 'BaseLogger', log_file: str = 'default.log', log_level: str = 'error') -> None: + """ + Initializes the BaseLogger with optional name, log file, and log level. + + Parameters: + name (str): The name of the logger. + log_file (str): The name of the file to log messages to. + log_level (str): The initial log level for the file handler. + """ + self.config = Config() + self.logger = logging.getLogger(name) + self.log_folder = self.config.data['settings']['system']['logging']['folder'] + self.log_file = log_file + + if not self.config.data['settings']['system']['logging']['enabled']: + self.logger.setLevel(logging.CRITICAL + 1) # Disable logging + return + + file_level = self._get_level_code(log_level) + console_level = self._get_level_code( + self.config.data['settings']['system']['logging'].get('console_level', 'warning') + ) + self.logger.setLevel(min(file_level, console_level)) + + self._setup_file_handler(file_level) + self._setup_console_handler(console_level) + + @staticmethod + def _get_level_code(level: str) -> int: + """ + Converts a log level as a string to the corresponding logging module level code. + + Parameters: + level (str): The log level as a string (e.g., 'debug', 'info', 'warning', 'error', 'critical'). + + Returns: + int: The logging module level code corresponding to the provided string. + """ + level_dict = { + 'debug': logging.DEBUG, + 'info': logging.INFO, + 'warning': logging.WARNING, + 'error': logging.ERROR, + 'critical': logging.CRITICAL, + } + return level_dict.get(level.lower(), logging.INFO) + + def _setup_console_handler(self, level: int) -> None: + """ + Sets up a console handler for logging messages to the console. Configures logging format and level. + + Parameters: + level (int): The logging level to set for the console handler. + """ + formatter = ColoredFormatter('%(levelname)s: %(message)s') + + if self.logger.name in BaseLogger.console_handlers: + ch = BaseLogger.console_handlers[self.logger.name] + if ch not in self.logger.handlers: + ch.setLevel(level) + ch.setFormatter(formatter) + self.logger.addHandler(ch) + return + + ch = logging.StreamHandler() + ch.setLevel(level) + ch.setFormatter(formatter) + self.logger.addHandler(ch) + BaseLogger.console_handlers[self.logger.name] = ch + + def _setup_file_handler(self, level: int) -> None: + """ + Sets up a file handler for logging messages to a file. Initializes the log folder and file if they do not exist, + and configures logging format and level. + + Parameters: + level (int): The logging level to set for the file handler. + """ + self._initialize_logging() + + formatter = logging.Formatter( + '%(asctime)s - %(levelname)s - %(message)s\n-------------------------------------------------------------', + datefmt='%Y-%m-%d %H:%M:%S' + ) + + if self.log_file in BaseLogger.file_handlers: + fh = BaseLogger.file_handlers[self.log_file] + if fh not in self.logger.handlers: + fh.setLevel(level) + fh.setFormatter(formatter) + self.logger.addHandler(fh) + return + + log_file_path = os.path.join(self.log_folder, self.log_file) + fh = logging.FileHandler(log_file_path, encoding='utf-8') + fh.setLevel(level) + fh.setFormatter(formatter) + self.logger.addHandler(fh) + BaseLogger.file_handlers[self.log_file] = fh + + def _initialize_logging(self) -> None: + """ + Initializes logging by ensuring the log folder exists. + """ + if not os.path.exists(self.log_folder): + os.makedirs(self.log_folder) + + def log_msg(self, msg: str, level: str = 'info') -> None: + """ + Logs a message at the specified log level. + + Parameters: + msg (str): The message to log. + level (str): The level at which to log the message (e.g., 'info', 'debug', 'error'). + """ + level_code = self._get_level_code(level) + self.logger.log(level_code, msg) + + def set_level(self, level: str) -> None: + """ + Sets the log level for the logger and its handlers. + + Parameters: + level (str): The new log level to set (e.g., 'info', 'debug', 'error'). + """ + level_code = self._get_level_code(level) + self.logger.setLevel(level_code) + for handler in self.logger.handlers: + handler.setLevel(level_code) + + +class Logger: + """ + A wrapper class for managing multiple BaseLogger instances, supporting different log files and levels + as configured in the system settings. + + This class facilitates logging across different modules and components of the application, allowing + for specific logs for agent activities, model interactions, and results. + + Attributes: + _instances (dict): A dictionary of Logger instances keyed by name. + loggers (dict): A dictionary of BaseLogger instances keyed by log type. + """ + + _instances = {} + _lock = threading.Lock() # Class-level lock for thread safety + VALID_LOGGER_NAME_PATTERN = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_]*$') + + def __new__(cls, name: str, default_logger: str = 'agentforge'): + """ + Create a new instance of Logger if one doesn't exist, or return the existing instance. + + Parameters: + name (str): The name of the module or component using the logger. + default_logger (str): The default logger file to use. + """ + with cls._lock: + if name not in cls._instances: + instance = super(Logger, cls).__new__(cls) + cls._instances[name] = instance + instance._initialized = False + return cls._instances[name] + + def __init__(self, name: str, default_logger: str = 'agentforge') -> None: + """ + Initializes the Logger class with names for different types of logs. + Initialization will only happen once. + + Parameters: + name (str): The name of the module or component using the logger. + default_logger (str): The default logger file to use. + """ + + if self._initialized: + return + + with Logger._lock: + self.config = Config() + self.caller_name = name # Stores the __name__ of the script that instantiated the Logger + self.default_logger = default_logger + self.logging_config = None + self.loggers = {} + + self.load_logging_config() + + self.update_logger_config(default_logger) + self.init_loggers() + + self._initialized = True + + def load_logging_config(self): + self.logging_config = self.config.data['settings']['system']['logging']['files'] + + def update_logger_config(self, logger_file: str): + """ + Adds a new logger to the configuration if it doesn't exist. + + Parameters: + logger_file (str): The name of the logger file to add. + """ + if logger_file and not self.VALID_LOGGER_NAME_PATTERN.match(logger_file): + raise ValueError( + f"Invalid logger_file name: '{logger_file}'. Must match pattern: {self.VALID_LOGGER_NAME_PATTERN.pattern}") + + with Logger._lock: + if logger_file not in self.logging_config: + self.logging_config[logger_file] = 'warning' + self.config.save() + + def init_loggers(self): + # Initialize loggers dynamically based on configuration settings + self.loggers = {} + for logger_file, log_level in self.logging_config.items(): + self.create_logger(logger_file, log_level) + + def create_logger(self, logger_file: str, log_level: str = 'warning'): + with Logger._lock: + logger_name = f'{self.caller_name}.{logger_file}' + log_file_name = f'{logger_file}.log' + new_logger = BaseLogger(name=logger_name, log_file=log_file_name, log_level=log_level) + self.loggers[logger_file] = new_logger + + def log(self, msg: str, level: str = 'info', logger_file: str = None) -> None: + """ + Logs a message to a specified logger. + + Parameters: + msg (str): The message to log. + level (str): The log level (e.g., 'info', 'debug', 'error'). + logger_file (str): The specific logger to use. If None, uses the default logger. + """ + # Prepend the caller's module name to the log message + msg_with_caller = f'[{self.caller_name}] {msg}' + + if logger_file is None: + logger_file = self.default_logger + + if logger_file not in self.loggers: + self.update_logger_config(logger_file) + self.create_logger(logger_file) + + logger = self.loggers.get(logger_file) + if logger: + logger.log_msg(msg_with_caller, level) + return + + raise ValueError(f"Logger '{logger_file}' could not be created.") + + def debug(self, msg: str, logger_file: str = None) -> None: + """Logs a debug level message.""" + self.log(msg, level='debug', logger_file=logger_file) + + def info(self, msg: str, logger_file: str = None) -> None: + """Logs an info level message.""" + self.log(msg, level='info', logger_file=logger_file) + + def warning(self, msg: str, logger_file: str = None) -> None: + """Logs a warning level message.""" + self.log(msg, level='warning', logger_file=logger_file) + + def error(self, msg: str, logger_file: str = None) -> None: + """Logs an error level message.""" + self.log(msg, level='error', logger_file=logger_file) + + def critical(self, msg: str, logger_file: str = None) -> None: + """Logs a critical level message.""" + self.log(msg, level='critical', logger_file=logger_file) + + def log_prompt(self, model_prompt: dict) -> None: + """ + Logs a prompt to the model interaction logger. + + Parameters: + model_prompt (dict): A dictionary containing the model prompts. + """ + system_prompt = model_prompt.get('system', '') + user_prompt = model_prompt.get('user', '') + msg = ( + f'******\nSystem Prompt\n******\n{system_prompt}\n' + f'******\nUser Prompt\n******\n{user_prompt}\n' + f'******' + ) + self.debug(msg, logger_file='model_io') + + def log_response(self, response: str) -> None: + """ + Logs a model response to the model interaction logger. + + Parameters: + response (str): The model response to log. + """ + msg = f'******\nModel Response\n******\n{response}\n******' + self.debug(msg, logger_file='model_io') + + def parsing_error(self, model_response: str, error: Exception) -> None: + """ + Logs parsing errors along with the model response. + + Parameters: + model_response (str): The model response associated with the parsing error. + error (Exception): The exception object representing the parsing error. + """ + msg = ( + f"Parsing Error - The model may not have responded in the required format.\n\n" + f"Model Response:\n******\n{model_response}\n******\n\nError: {error}" + ) + self.error(msg) \ No newline at end of file diff --git a/src/agentforge/utils/parsing_processor.py b/src/agentforge/utils/parsing_processor.py new file mode 100644 index 00000000..29cf50f3 --- /dev/null +++ b/src/agentforge/utils/parsing_processor.py @@ -0,0 +1,237 @@ +import re +import json +import yaml +from typing import Optional, Dict, Any, Callable, Type, List +from agentforge.utils.logger import Logger +import xmltodict +import configparser +import csv +from io import StringIO + +class ParsingProcessor: + + def __init__(self): + """ + Initializes the ParsingUtils class. + """ + # Assuming Logger is defined elsewhere or replace with appropriate logging + self.logger = Logger(name=self.__class__.__name__) + + def extract_code_block(self, text: str) -> Optional[tuple[Optional[str], str]]: + """ + Extracts the content of a code block from a string and returns the language specifier if present. + Supports code blocks with or without a language specifier. + If multiple code blocks are present, returns the first one. + If no code block is found, returns the input text and None. + + Parameters: + text (str): The text containing the code block. + + Returns: + Optional[Tuple[Optional[str], str]]: A tuple containing the code block content and the language specifier (or None). + """ + try: + # Updated regex pattern with a greedy match + code_block_pattern = r"```([a-zA-Z]*)\r?\n([\s\S]*?)```" + match = re.search(code_block_pattern, text, re.DOTALL) + + if match: + language = match.group(1).strip() or None + content = match.group(2).strip() + # return content, language + return language, content + + # If no code block is found, return the input text and None as the language + return None, text.strip() + except Exception as e: + self.logger.log(f"Regex Error Extracting Code Block: {e}", 'error') + return None + + def parse_content( + self, + content_string: str, + parser_func: Callable[[str], Any], + expected_language: str, + exception_class: Type[Exception] + ) -> Optional[Dict[str, Any]]: + """ + A generic method to parse content using a specified parser function. + + Parameters: + content_string (str): The string containing the content to parse. + parser_func (Callable[[str], Any]): The parsing function (e.g., json.loads, yaml.safe_load). + expected_language (str): The expected language specifier (e.g., 'json', 'yaml'). + exception_class (Type[Exception]): The exception class to catch during parsing. + + Returns: + Optional[Dict[str, Any]]: The parsed content as a dictionary, or None if parsing fails. + """ + try: + language, cleaned_string = self.extract_code_block(content_string) + if language and language.lower() != expected_language.lower(): + self.logger.log(f"Expected {expected_language.upper()} code block, but found '{language}'", 'warning') + + if cleaned_string: + return parser_func(cleaned_string) + return None + except exception_class as e: + self.logger.log(f"Parsing error: {e}", 'error') + return None + except Exception as e: + self.logger.log(f"Unexpected error parsing {expected_language.upper()} content: {e}", 'error') + return None + + @staticmethod + def parse_markdown_to_dict(markdown_text: str, min_heading_level=2, max_heading_level=6) -> Optional[Dict[str, Any]]: + """ + Parses a markdown-formatted string into a dictionary, mapping each heading to its corresponding content. + + Parameters: + markdown_text (str): The markdown-formatted text to parse. + min_heading_level (int, optional): The minimum heading level to include (default is 2). + max_heading_level (int, optional): The maximum heading level to include (default is 6). + + Returns: + Optional[Dict[str, Any]]: A dictionary where each key is a heading and each value is the associated content. + """ + parsed_dict = {} + current_heading = None + content_lines = [] + + # Compile regex pattern for headings based on specified heading levels + heading_pattern = re.compile(r'^(#{%d,%d})\s+(.*)' % (min_heading_level, max_heading_level)) + + lines = markdown_text.split('\n') + for line in lines: + match = heading_pattern.match(line) + if match: + # Save content under the previous heading + if current_heading is not None: + parsed_dict[current_heading] = '\n'.join(content_lines).strip() + content_lines = [] + # Update current heading + current_heading = match.group(2).strip() + else: + if current_heading is not None: + content_lines.append(line) + # Save content under the last heading + if current_heading is not None: + parsed_dict[current_heading] = '\n'.join(content_lines).strip() + + return parsed_dict if parsed_dict else None + + def parse_markdown_content(self, markdown_string: str, min_heading_level=2, max_heading_level=6) -> Optional[Dict[str, Any]]: + """ + Parses a Markdown-formatted string into a Python dictionary. + + Parameters: + markdown_string (str): The Markdown string to parse. + min_heading_level (int, optional): The minimum heading level to include (default is 2). + max_heading_level (int, optional): The maximum heading level to include (default is 6). + + Returns: + Optional[Dict[str, Any]]: The parsed Markdown content as a dictionary, or None if parsing fails. + """ + def parser_func(s): + return self.parse_markdown_to_dict(s, min_heading_level, max_heading_level) + + return self.parse_content( + content_string=markdown_string, + parser_func=parser_func, + expected_language='markdown', + exception_class=Exception + ) + + def parse_yaml_content(self, yaml_string: str) -> Optional[Dict[str, Any]]: + """ + Parses a YAML-formatted string into a Python dictionary. + + Parameters: + yaml_string (str): The YAML string to parse. + + Returns: + Optional[Dict[str, Any]]: The parsed YAML content as a dictionary, or None if parsing fails. + """ + return self.parse_content( + content_string=yaml_string, + parser_func=yaml.safe_load, + expected_language='yaml', + exception_class=yaml.YAMLError + ) + + def parse_json_content(self, json_string: str) -> Optional[Dict[str, Any]]: + """ + Parses a JSON-formatted string into a Python dictionary. + + Parameters: + json_string (str): The JSON string to parse. + + Returns: + Optional[Dict[str, Any]]: The parsed JSON content as a dictionary, or None if parsing fails. + """ + return self.parse_content( + content_string=json_string, + parser_func=json.loads, + expected_language='json', + exception_class=json.JSONDecodeError + ) + + def parse_xml_content(self, xml_string: str) -> Optional[Dict[str, Any]]: + """ + Parses an XML-formatted string into a Python dictionary. + + Parameters: + xml_string (str): The XML string to parse. + + Returns: + Optional[Dict[str, Any]]: The parsed XML content as a dictionary, or None if parsing fails. + """ + return self.parse_content( + content_string=xml_string, + parser_func=xmltodict.parse, + expected_language='xml', + exception_class=Exception # Replace with a more specific exception if available + ) + + def parse_ini_content(self, ini_string: str) -> Optional[Dict[str, Any]]: + """ + Parses an INI-formatted string into a Python dictionary. + + Parameters: + ini_string (str): The INI string to parse. + + Returns: + Optional[Dict[str, Any]]: The parsed INI content as a dictionary, or None if parsing fails. + """ + def parser_func(s): + parser = configparser.ConfigParser() + parser.read_string(s) + return {section: dict(parser.items(section)) for section in parser.sections()} + + return self.parse_content( + content_string=ini_string, + parser_func=parser_func, + expected_language='ini', + exception_class=configparser.Error + ) + + def parse_csv_content(self, csv_string: str) -> Optional[List[Dict[str, Any]]]: + """ + Parses a CSV-formatted string into a list of dictionaries. + + Parameters: + csv_string (str): The CSV string to parse. + + Returns: + Optional[List[Dict[str, Any]]]: The parsed CSV content as a list of dictionaries, or None if parsing fails. + """ + def parser_func(s): + reader = csv.DictReader(StringIO(s)) + return [row for row in reader] + + return self.parse_content( + content_string=csv_string, + parser_func=parser_func, + expected_language='csv', + exception_class=csv.Error + ) diff --git a/src/agentforge/utils/PromptHandling.py b/src/agentforge/utils/prompt_processor.py similarity index 94% rename from src/agentforge/utils/PromptHandling.py rename to src/agentforge/utils/prompt_processor.py index 3e5b3958..630e1424 100755 --- a/src/agentforge/utils/PromptHandling.py +++ b/src/agentforge/utils/prompt_processor.py @@ -1,8 +1,8 @@ import re -from agentforge.utils.Logger import Logger +from agentforge.utils.logger import Logger -class PromptHandling: +class PromptProcessor: """ A utility class for handling dynamic prompt templates. It supports extracting variables from templates, checking for the presence of required variables in data, and rendering templates with values from provided data. @@ -33,16 +33,16 @@ def check_prompt_format(self, prompts): or if the sub-prompts are not dictionaries. """ # Check if 'System' and 'User' are the only keys present - if set(prompts.keys()) != {'System', 'User'}: + if set(prompts.keys()) != {'system', 'user'}: error_message = ( - "Error: Prompts should contain only 'System' and 'User' keys. " + "Error: Prompts should contain only 'system' and 'user' keys. " "Please check the prompt YAML file format." ) self.logger.log(error_message, 'error') raise ValueError(error_message) # Allow 'System' and 'User' prompts to be either dicts or strings - for prompt_type in ['System', 'User']: + for prompt_type in ['system', 'user']: prompt_value = prompts.get(prompt_type, {}) if not isinstance(prompt_value, (dict, str)): error_message = ( @@ -149,7 +149,7 @@ def render_prompts(self, prompts, data): """ try: rendered_prompts = {} - for prompt_type in ['System', 'User']: + for prompt_type in ['system', 'user']: rendered_sections = [] prompt_content = prompts.get(prompt_type, {}) if isinstance(prompt_content, str): diff --git a/src/agentforge/utils/ToolUtils.py b/src/agentforge/utils/tool_utils.py similarity index 89% rename from src/agentforge/utils/ToolUtils.py rename to src/agentforge/utils/tool_utils.py index 2881c6a8..db5b0376 100755 --- a/src/agentforge/utils/ToolUtils.py +++ b/src/agentforge/utils/tool_utils.py @@ -1,11 +1,11 @@ -# utils/functions/ToolUtils.py +# utils/functions/tool_utils.py import traceback import importlib from typing import List, Optional, Union -from agentforge.utils.Logger import Logger +from agentforge.utils.logger import Logger from typing import Any, Dict -from agentforge.utils.ChromaUtils import ChromaUtils +from agentforge.storage.chroma_storage import ChromaStorage class ToolUtils: @@ -31,7 +31,7 @@ def __init__(self): Initializes the ToolUtils class with a Logger instance. """ self.logger = Logger(name=self.__class__.__name__) - self.storage = ChromaUtils('default') + self.storage = ChromaStorage('default') # -------------------------------------------------------------------------------------------------------- # ----------------------------------------- Dynamic Tool Methods ----------------------------------------- @@ -50,10 +50,10 @@ def dynamic_tool(self, tool: Dict[str, str], payload: Dict[str, Any]) -> Dict[st dict: The result of executing the command within the tool, or an error dictionary if an error occurs. """ tool_module = tool.get('Script') - tool_class = tool_module.split('.')[-1] + tool_class = tool.get('Class') command = tool.get('Command') args = payload['args'] - self.logger.log_info(f"\nRunning {tool_class} ...") + self.logger.info(f"\nRunning {tool_class} ...") try: result = self._execute_tool(tool_module, tool_class, command, args) @@ -79,7 +79,12 @@ def _execute_tool(self, tool_module: str, tool_class: str, command: str, args: D command_func = self.BUILTIN_FUNCTIONS[tool_module] # type: ignore result = command_func(**args) else: - tool = importlib.import_module(tool_module) + if tool_module.startswith('.agentforge'): + # Remove '.agentforge' from the beginning of the path + relative_path = tool_module.replace('.agentforge', '', 1) + tool = importlib.import_module(relative_path, package='agentforge') + else: + tool = importlib.import_module(tool_module) if hasattr(tool, tool_class): tool_instance = getattr(tool, tool_class)() command_func = getattr(tool_instance, command) diff --git a/tests/agent_tests/test_agent.py b/tests/agent_tests/test_agent.py new file mode 100644 index 00000000..dab7cdc3 --- /dev/null +++ b/tests/agent_tests/test_agent.py @@ -0,0 +1,18 @@ +import unittest +from agentforge.agent import Agent +from agentforge.config import Config + +class TestAgent(unittest.TestCase): + # Provide an explicit path to .agentforge (within your test directory or setup_files) + test_root = "../src/agentforge/setup_files" # Adjust as needed + config_override = Config(root_path=test_root) + + def test_agent_init(self): + agent = Agent(agent_name="TestAgent") + self.assertEqual(agent.agent_name, "TestAgent") + self.assertIsNotNone(agent.logger) + self.assertIsNotNone(agent.config) + self.assertIsNotNone(agent.prompt_processor) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/base_test_case.py b/tests/base_test_case.py new file mode 100644 index 00000000..6080152c --- /dev/null +++ b/tests/base_test_case.py @@ -0,0 +1,96 @@ +# base_test_case.py + +import unittest +import tempfile +import shutil +from pathlib import Path +from unittest.mock import patch +from agentforge.config import Config + +# --------------------------------- +# Prep. +# --------------------------------- + +class BaseTestCaseNoAgentForgeFolder(unittest.TestCase): + + # --------------------------------- + # Prep. + # --------------------------------- + + def setUp(self): + Config._instance = None + self._patch_print() + self._create_temp_directory() + + def tearDown(self): + Config._instance = None + self.temp_dir.cleanup() + + # --------------------------------- + # Internal Methods + # --------------------------------- + + def _patch_print(self): + # Supress Print Statements + self.print_patch = patch("builtins.print", lambda *args, **kwargs: None) + self.print_patch.start() + + def _create_temp_directory(self): + # Create a temporary directory to copy the real .agentforge folder into + self.temp_dir = tempfile.TemporaryDirectory() + self.temp_root_path = Path(self.temp_dir.name) + + +class BaseTestCaseEmptyAgentForgeFolder(BaseTestCaseNoAgentForgeFolder): + + # --------------------------------- + # Prep. + # --------------------------------- + + def setUp(self): + super().setUp() + self._create_agentforge_folder() + + # --------------------------------- + # Internal Methods + # --------------------------------- + + def _create_agentforge_folder(self): + # Create an empty .agentforge folder + (self.temp_root_path / ".agentforge").mkdir(exist_ok=True) + + +class BaseTestCase(BaseTestCaseNoAgentForgeFolder): + + # --------------------------------- + # Prep. + # --------------------------------- + + def setUp(self): + super().setUp() + self._copy_agentforge_files() + self._reset_config() + + def tearDown(self): + super().setUp() + self.config = None + + # --------------------------------- + # Internal Methods + # --------------------------------- + + def _copy_agentforge_files(self): + # Copy the existing .agentforge from setup_files into the temp dir + # If some classes need an empty .agentforge, override or skip this + root_dir = Path(__file__).resolve().parent + real_agentforge = root_dir.parent / "src" / "agentforge" / "setup_files" / ".agentforge" + shutil.copytree(real_agentforge, self.temp_root_path / ".agentforge") + + def _reset_config(self): + # Reset the Config singleton with the desired temp root path + self.config = Config.reset(root_path=str(self.temp_root_path)) + +if __name__ == '__main__': + unittest.main() + + diff --git a/tests/config_tests/test_config_defaults.py b/tests/config_tests/test_config_defaults.py new file mode 100644 index 00000000..f1774679 --- /dev/null +++ b/tests/config_tests/test_config_defaults.py @@ -0,0 +1,111 @@ +import unittest +from tests.base_test_case import BaseTestCase # Adjust as needed for your framework + +class TestConfigDefaults(BaseTestCase): + + # --------------------------------- + # Prep. + # --------------------------------- + + def setUp(self): + super().setUp() + self.system_data = self.config.data['settings']['system'] + self.models_data = self.config.data['settings']['models'] + self.storage_data = self.config.data['settings']['storage'] + + def tearDown(self): + super().tearDown() + self.system_data = None + + # --------------------------------- + # Test System Defaults. + # --------------------------------- + + def test_default_persona_settings(self): + # Persona settings + persona_data = self.system_data.get('persona', {}) + self.assertTrue(persona_data.get('enabled', False)) + self.assertEqual(persona_data.get('name'), 'default') + + def test_default_debug_settings(self): + # Debug settings + debug_data = self.system_data.get('debug', {}) + self.assertFalse(debug_data.get('mode', True)) + self.assertFalse(debug_data.get('save_memory', True)) + self.assertEqual( + debug_data.get('simulated_response'), + "Text designed to simulate an LLM response for debugging purposes without invoking the model." + ) + + def test_default_logging_settings(self): + # Logging settings + logging_data = self.system_data.get('logging', {}) + self.assertTrue(logging_data.get('enabled', False)) + self.assertEqual(logging_data.get('folder'), './logs') + + log_files = logging_data.get('files', {}) + self.assertEqual(log_files.get('agentforge'), 'error') + self.assertEqual(log_files.get('model_io'), 'error') + + def test_default_misc_settings(self): + # Misc settings + misc_data = self.system_data.get('misc', {}) + self.assertTrue(misc_data.get('on_the_fly', False)) + + def test_default_path_settings(self): + # Paths + paths_data = self.system_data.get('paths', {}) + self.assertEqual(paths_data.get('files'), './files') + + # --------------------------------- + # Test Model Defaults. + # --------------------------------- + + def test_default_model_settings(self): + # Default model settings + default_model = self.models_data.get('default_model', {}) + self.assertEqual(default_model.get('api'), 'gemini_api') + self.assertEqual(default_model.get('model'), 'gemini_flash') + + def test_default_model_library_settings(self): + # Model library validations + model_library = self.models_data.get('model_library', {}) + self.assertIn('openai_api', model_library) + self.assertIn('gemini_api', model_library) + self.assertIn('lm_studio_api', model_library) + + gemini_api = model_library.get('gemini_api', {}).get('Gemini', {}).get('models', {}) + self.assertIn('gemini_pro', gemini_api) + self.assertIn('gemini_flash', gemini_api) + + gemini_params = model_library.get('gemini_api', {}).get('Gemini', {}).get('params', {}) + self.assertEqual(gemini_params.get('max_output_tokens'), 10000) + self.assertEqual(gemini_params.get('temperature'), 0.8) + + # --------------------------------- + # Test Storage Defaults. + # --------------------------------- + + def test_default_option_settings(self): + # Options + options_data = self.storage_data.get('options', {}) + self.assertTrue(options_data.get('enabled', False)) + self.assertTrue(options_data.get('save_memory', False)) + self.assertTrue(options_data.get('iso_timestamp', False)) + self.assertTrue(options_data.get('unix_timestamp', False)) + self.assertEqual(options_data.get('persist_directory'), './db/ChromaDB') + self.assertFalse(options_data.get('fresh_start', True)) + + def test_default_embedding_settings(self): + # Embedding settings + embedding_data = self.storage_data.get('embedding', {}) + self.assertEqual(embedding_data.get('selected'), 'distil_roberta') + + def test_default_embedding_library_settings(self): + # Library defaults + library_data = self.storage_data.get('embedding_library', {}) + self.assertEqual(library_data.get('distil_roberta'), 'all-distilroberta-v1') + self.assertEqual(library_data.get('all_mini'), 'all-MiniLM-L6-v2') + +if __name__ == '__main__': + unittest.main() diff --git a/tests/config_tests/test_config_integration.py b/tests/config_tests/test_config_integration.py new file mode 100644 index 00000000..d4fd46e3 --- /dev/null +++ b/tests/config_tests/test_config_integration.py @@ -0,0 +1,42 @@ +# test_config_integration.py + +import unittest +from agentforge.config import Config +from tests.base_test_case import BaseTestCase, BaseTestCaseEmptyAgentForgeFolder, BaseTestCaseNoAgentForgeFolder + + +class TestConfigIntegrationNoAgentForge(BaseTestCaseNoAgentForgeFolder): + + def test_no_agentforge_folder_raises(self): + with self.assertRaises(FileNotFoundError): + Config(root_path=str(self.temp_root_path)) + + +class TestConfigIntegrationEmptyAgentForge(BaseTestCaseEmptyAgentForgeFolder): + + def test_empty_agentforge_loads_no_settings(self): + config = Config(root_path=self.temp_root_path) + self.assertEqual(config.data, {}, "Expected an empty config.data but got something else") + +class TestConfigIntegration(BaseTestCase): + + def test_custom_root_path(self): + # Assert known configs from real .agentforge is loaded + self.assertIn("actions", self.config.data) + self.assertIn("personas", self.config.data) + self.assertIn("prompts", self.config.data) + self.assertIn("settings", self.config.data) + self.assertIn("tools", self.config.data) + + # Assert Setting Files are loaded + self.assertIn("models", self.config.data["settings"]) + self.assertIn("storage", self.config.data["settings"]) + self.assertIn("system", self.config.data["settings"]) + + # Assert settings are set to the correct default value + self.assertTrue(self.config.data["settings"]["system"]["persona"].get("enabled")) + + # ... maybe we write a separate test for validating a system, storage and models files ... + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/config_tests/test_config_persona.py b/tests/config_tests/test_config_persona.py new file mode 100644 index 00000000..57fe098d --- /dev/null +++ b/tests/config_tests/test_config_persona.py @@ -0,0 +1,44 @@ +# test_config_persona.py +import unittest +from tests.base_test_case import BaseTestCase + +class TestLoadPersona(BaseTestCase): + + def test_personas_disabled_returns_none(self): + self.config.data['settings']['system']['persona']['enabled'] = False + + agent_data = {} # no explicit Persona key + persona = self.config.load_persona(agent_data) + self.assertIsNone(persona, "Expected None when persona is disabled.") + + def test_missing_persona_raises_file_not_found(self): + # Make sure persona is enabled + self.config.data['settings']['system']['persona']['enabled'] = True + self.config.data['personas'] = {} # or ensure the requested persona doesn't exist + + with self.assertRaises(FileNotFoundError): + self.config.load_persona({'Persona': 'NonExistentPersona'}) + + def test_load_default_persona(self): + # Enable persona + self.config.data['settings']['system']['persona']['enabled'] = True + # Suppose it's named 'default' + self.config.data['settings']['system']['persona']['name'] = 'default' + + # Make sure there's an actual 'default' persona in the data + self.config.data['personas'] = { + 'default': {"some_key": "some_value"} + } + + persona = self.config.load_persona({}) + self.assertIsNotNone(persona) + self.assertEqual(persona.get('some_key'), "some_value") + + def test_load_explicit_persona(self): + self.config.data['settings']['system']['persona']['enabled'] = True + + persona = self.config.load_persona(self.config.data) + self.assertEqual(persona.get('Name'), "Persona Name") + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/config_tests/test_config_unit.py b/tests/config_tests/test_config_unit.py new file mode 100644 index 00000000..24216c57 --- /dev/null +++ b/tests/config_tests/test_config_unit.py @@ -0,0 +1,47 @@ +# test_config_unit.py + +import unittest +from unittest.mock import patch, MagicMock +from agentforge.config import Config +from tests.base_test_case import BaseTestCase, BaseTestCaseEmptyAgentForgeFolder, BaseTestCaseNoAgentForgeFolder + +class TestConfigNoAgentForge(BaseTestCaseNoAgentForgeFolder): + + def test_config_raises_FileNotFound_for_missing_folder(self): + # temp_root_path is a custom root path that does NOT contain .agentforge + # We expect a FileNotFoundError to be raised + with self.assertRaises(FileNotFoundError): + Config(root_path=str(self.temp_root_path)) + + +class TestEmptyConfig(BaseTestCaseEmptyAgentForgeFolder): + + @patch.object(Config, 'find_project_root') + def test_config_without_root_path(self, mock_find_project_root): + # Instead of letting it do the real search, we'll just ensure it *calls* find_project_root + # then we’ll simulate returning the same directory we set up in setUp. + mock_find_project_root.return_value = self.temp_root_path.resolve() + + config = Config() + + mock_find_project_root.assert_called_once() + self.assertEqual(config.project_root, self.temp_root_path.resolve()) + self.assertTrue(config.config_path.is_dir()) + + +class TestConfig(BaseTestCase): + + def test_config_with_custom_root_path(self): + # Ensure that the discovered project_root is exactly our temp directory + self.assertEqual(self.config.project_root, self.temp_root_path.resolve()) + self.assertTrue(self.config.config_path.is_dir(), "config_path should point to an existing .agentforge directory") + + @patch.object(Config, 'reload', return_value=None) + def test_load_agent_data_agent_not_found(self, mock_reload): + # We'll raise FileNotFoundError if the agent is not found + with self.assertRaises(FileNotFoundError): + self.config.load_agent_data("MissingAgent") + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/storage_tests/test_chroma_storage.py b/tests/storage_tests/test_chroma_storage.py new file mode 100644 index 00000000..f75febed --- /dev/null +++ b/tests/storage_tests/test_chroma_storage.py @@ -0,0 +1,119 @@ +# test_chroma_storage.py + +from agentforge.storage.chroma_storage import ChromaStorage + +import unittest +from tests.base_test_case import BaseTestCase + +class TestChromaStorage(BaseTestCase): + + # --------------------------------- + # Prep. + # --------------------------------- + + def setUp(self): + super().setUp() + self.storage = ChromaStorage() + + def tearDown(self): + super().tearDown() + # Clear ONLY IF the chroma client exists otherwise no need to do anything + if self.storage.client: + self.storage.reset_storage() + self.storage.disconnect() + + self.storage = None + + # --------------------------------- + # Tests + # --------------------------------- + + def test_connect_disconnect(self): + """ + Verifies we can connect to Chroma and then disconnect without errors. + """ + self.storage.connect() + # Maybe we can check some internal state or call a method that verifies a connection. + # But for now, we might just see if it doesn't raise any exceptions. + + self.storage.disconnect() + # Same idea: if there's no exception, that might suffice for an initial test. + # But you can check state changes if needed. + + def test_create_and_delete_collection(self): + """ + Basic test to confirm that we can create a collection (table) and delete it. + """ + self.storage.connect() + self.storage.create_collection("test_collection") + # Potentially check that the collection actually exists by some method + self.storage.delete_collection("test_collection") + # Check that it actually got removed or handle exceptions as needed. + + def test_insert_and_query(self): + """ + Test that we can insert documents into a collection and retrieve them with a query. + """ + self.storage.connect() + + self.storage.create_collection("test_collection") + + # Insert one or more documents + data_to_insert = ["Hello"] + self.storage.insert("test_collection", ids=["1"], data=data_to_insert) + + # Query them back + results = self.storage.query(collection_name="test_collection", ids=["1"]) + self.assertTrue(len(results['documents']) == 1) + self.assertEqual(results['documents'][0], "Hello") + + def test_update_and_query(self): + """ + Insert, then update a record, then check if we get the updated field back. + """ + self.storage.connect() + self.storage.create_collection("test_collection") + + # Insert documents + self.storage.insert("test_collection", ids=["1"], data=['Foo']) + self.storage.update("test_collection", ids=["1"], new_data=['Bar']) + + results = self.storage.query(collection_name="test_collection", ids=["1"]) + self.assertTrue(len(results['documents']) == 1) + self.assertEqual(results['documents'][0], "Bar") + + def test_delete_and_count(self): + """ + Insert some records, delete some of them, and count how many remain. + """ + self.storage.connect() + self.storage.create_collection("test_collection") + + # Insert documents + self.storage.insert("test_collection", ids=["1", "2", "3"], data=['Foo', 'Bar', 'Bacon']) + self.storage.delete("test_collection", ids=["2"]) + + # count should be 2 + count_after_delete = self.storage.count("test_collection") + self.assertEqual(count_after_delete, 2) + + def test_reset_storage(self): + """ + After inserting data, reset_storage should nuke it all. + """ + self.storage.connect() + self.storage.create_collection("test_collection") + self.storage.insert("test_collection", ids=["1", "2"], data=['Foo', 'Bar']) + + count_before_reset = self.storage.count("test_collection") + self.assertEqual(count_before_reset, 2) + + self.storage.reset_storage() + + # reset_storage deletes the entire DB, so test_collection will not exist + # We'll test the simplest case: everything is gone. + self.assertRaises(Exception, self.storage.count, "test_collection") + + +if __name__ == "__main__": + unittest.main()