From 824f584185c62d4a1c898fb691800cbfb83fb126 Mon Sep 17 00:00:00 2001 From: michaeldecent2 <111002205+MichaelDecent@users.noreply.github.com> Date: Tue, 19 Nov 2024 12:15:14 +0100 Subject: [PATCH 001/168] swarm - feat: add image generation interface and base implementation --- pkgs/core/swarmauri_core/ComponentBase.py | 1 + .../swarmauri_core/image_gens/IGenImage.py | 35 +++ .../swarmauri_core/image_gens/__init__.py | 0 .../swarmauri/swarmauri/image_gens/__init_.py | 0 .../swarmauri/image_gens/base/ImageGenBase.py | 51 ++++ .../swarmauri/image_gens/base/__init__.py | 0 .../concrete/BlackForestImgGenModel.py | 259 ++++++++++++++++++ .../swarmauri/image_gens/concrete/__init__.py | 0 .../BlackForestImgGenModel_unit_test.py | 120 ++++++++ 9 files changed, 466 insertions(+) create mode 100644 pkgs/core/swarmauri_core/image_gens/IGenImage.py create mode 100644 pkgs/core/swarmauri_core/image_gens/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/image_gens/__init_.py create mode 100644 pkgs/swarmauri/swarmauri/image_gens/base/ImageGenBase.py create mode 100644 pkgs/swarmauri/swarmauri/image_gens/base/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/image_gens/concrete/BlackForestImgGenModel.py create mode 100644 pkgs/swarmauri/swarmauri/image_gens/concrete/__init__.py create mode 100644 pkgs/swarmauri/tests/unit/image_gens/BlackForestImgGenModel_unit_test.py diff --git a/pkgs/core/swarmauri_core/ComponentBase.py b/pkgs/core/swarmauri_core/ComponentBase.py index 9f3e86e9f..4336b2047 100644 --- a/pkgs/core/swarmauri_core/ComponentBase.py +++ b/pkgs/core/swarmauri_core/ComponentBase.py @@ -34,6 +34,7 @@ class ResourceTypes(Enum): DOCUMENT = "Document" EMBEDDING = "Embedding" EXCEPTION = "Exception" + IMAGE_GEN = "ImageGen" LLM = "LLM" MESSAGE = "Message" MEASUREMENT = "Measurement" diff --git a/pkgs/core/swarmauri_core/image_gens/IGenImage.py b/pkgs/core/swarmauri_core/image_gens/IGenImage.py new file mode 100644 index 000000000..79cebf615 --- /dev/null +++ b/pkgs/core/swarmauri_core/image_gens/IGenImage.py @@ -0,0 +1,35 @@ +from abc import ABC, abstractmethod + + +class IGenImage(ABC): + """ + Interface focusing on the basic properties and settings essential for defining image generating models. + """ + + @abstractmethod + def generate_image(self, *args, **kwargs) -> any: + """ + Generate images based on the input data provided to the model. + """ + pass + + @abstractmethod + async def agenerate_image(self, *args, **kwargs) -> any: + """ + Generate images based on the input data provided to the model. + """ + pass + + @abstractmethod + def batch_generate(self, *args, **kwargs) -> any: + """ + Generate images based on the input data provided to the model. + """ + pass + + @abstractmethod + async def abatch_generate(self, *args, **kwargs) -> any: + """ + Generate images based on the input data provided to the model. + """ + pass diff --git a/pkgs/core/swarmauri_core/image_gens/__init__.py b/pkgs/core/swarmauri_core/image_gens/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/image_gens/__init_.py b/pkgs/swarmauri/swarmauri/image_gens/__init_.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/image_gens/base/ImageGenBase.py b/pkgs/swarmauri/swarmauri/image_gens/base/ImageGenBase.py new file mode 100644 index 000000000..e9fbda41e --- /dev/null +++ b/pkgs/swarmauri/swarmauri/image_gens/base/ImageGenBase.py @@ -0,0 +1,51 @@ +from abc import abstractmethod +from typing import Optional, List, Literal +from pydantic import ConfigDict, model_validator, Field +from swarmauri_core.image_gens.IGenImage import IGenImage +from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes + + +class ImageGenBase(IGenImage, ComponentBase): + allowed_models: List[str] = [] + resource: Optional[str] = Field(default=ResourceTypes.IMAGE_GEN.value, frozen=True) + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + type: Literal["LLMBase"] = "LLMBase" + + @model_validator(mode="after") + @classmethod + def _validate_name_in_allowed_models(cls, values): + name = values.name + allowed_models = values.allowed_models + if name and name not in allowed_models: + raise ValueError( + f"Model name {name} is not allowed. Choose from {allowed_models}" + ) + return values + + @abstractmethod + def generate_image(self, *args, **kwargs) -> any: + """ + Generate images based on the input data provided to the model. + """ + raise NotImplementedError("generate_image() not implemented in subclass yet.") + + @abstractmethod + async def agenerate_image(self, *args, **kwargs) -> any: + """ + Generate images based on the input data provided to the model. + """ + raise NotImplementedError("agenerate_image() not implemented in subclass yet.") + + @abstractmethod + def batch_generate(self, *args, **kwargs) -> any: + """ + Generate images based on the input data provided to the model. + """ + raise NotImplementedError("batch_generate() not implemented in subclass yet.") + + @abstractmethod + async def abatch_generate(self, *args, **kwargs) -> any: + """ + Generate images based on the input data provided to the model. + """ + raise NotImplementedError("abatch_generate() not implemented in subclass yet.") diff --git a/pkgs/swarmauri/swarmauri/image_gens/base/__init__.py b/pkgs/swarmauri/swarmauri/image_gens/base/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/image_gens/concrete/BlackForestImgGenModel.py b/pkgs/swarmauri/swarmauri/image_gens/concrete/BlackForestImgGenModel.py new file mode 100644 index 000000000..c697764ff --- /dev/null +++ b/pkgs/swarmauri/swarmauri/image_gens/concrete/BlackForestImgGenModel.py @@ -0,0 +1,259 @@ +import httpx +import time +from typing import List, Literal, Optional, Dict, ClassVar +from pydantic import PrivateAttr +from swarmauri.utils.retry_decorator import retry_on_status_codes +from swarmauri.image_gens.base.ImageGenBase import ImageGenBase +import asyncio +import contextlib + + +class BlackForestImgGenModel(ImageGenBase): + """ + A model for generating images using FluxPro's image generation models through the Black Forest API. + Link to API key: https://api.bfl.ml/auth/profile + """ + + _BASE_URL: str = PrivateAttr("https://api.bfl.ml") + _client: httpx.Client = PrivateAttr() + _async_client: httpx.AsyncClient = PrivateAttr(default=None) + + api_key: str + allowed_models: List[str] = ["flux-pro-1.1", "flux-pro", "flux-dev"] + + asyncio: ClassVar = asyncio + name: str = "flux-pro" # Default model + type: Literal["BlackForestImgGenModel"] = "BlackForestImgGenModel" + + def __init__(self, **data): + """ + Initializes the BlackForestImgGenModel instance with HTTP clients. + """ + super().__init__(**data) + self._headers = { + "Content-Type": "application/json", + "X-Key": self.api_key, + } + self._client = httpx.Client(headers=self._headers, timeout=30) + + async def _get_async_client(self) -> httpx.AsyncClient: + """Gets or creates an async client instance.""" + if self._async_client is None or self._async_client.is_closed: + self._async_client = httpx.AsyncClient(headers=self._headers, timeout=30) + return self._async_client + + async def _close_async_client(self): + """Closes the async client if it exists and is open.""" + if self._async_client is not None and not self._async_client.is_closed: + await self._async_client.aclose() + self._async_client = None + + @retry_on_status_codes((429, 529), max_retries=1) + def _send_request(self, endpoint: str, data: dict) -> dict: + """Send a synchronous request to FluxPro's API for image generation.""" + url = f"{self._BASE_URL}/{endpoint}" + response = self._client.post(url, json=data) + response.raise_for_status() + return response.json() + + @retry_on_status_codes((429, 529), max_retries=1) + async def _async_send_request(self, endpoint: str, data: dict) -> dict: + """Send an asynchronous request to FluxPro's API for image generation.""" + client = await self._get_async_client() + url = f"{self._BASE_URL}/{endpoint}" + response = await client.post(url, json=data) + response.raise_for_status() + return response.json() + + @retry_on_status_codes((429, 529), max_retries=1) + def _get_result(self, task_id: str) -> dict: + """Get the result of a generation task synchronously.""" + url = f"{self._BASE_URL}/v1/get_result" + params = {"id": task_id} + response = self._client.get(url, params=params) + response.raise_for_status() + return response.json() + + @retry_on_status_codes((429, 529), max_retries=1) + async def _async_get_result(self, task_id: str) -> dict: + """Get the result of a generation task asynchronously.""" + client = await self._get_async_client() + url = f"{self._BASE_URL}/v1/get_result" + params = {"id": task_id} + response = await client.get(url, params=params) + response.raise_for_status() + return response.json() + + def generate_image( + self, + prompt: str, + width: int = 1024, + height: int = 768, + steps: Optional[int] = None, + prompt_upsampling: bool = False, + seed: Optional[int] = None, + guidance: Optional[float] = None, + safety_tolerance: Optional[int] = None, + interval: Optional[float] = None, + max_wait_time: int = 300, + check_interval: int = 10, + ) -> Dict: + """ + Generates an image based on the prompt and waits for the result synchronously. + + Args: + prompt (str): The text prompt for image generation + width (int): Image width in pixels + height (int): Image height in pixels + steps (Optional[int]): Number of inference steps + prompt_upsampling (bool): Whether to use prompt upsampling + seed (Optional[int]): Random seed for generation + guidance (Optional[float]): Guidance scale + safety_tolerance (Optional[int]): Safety tolerance level + interval (Optional[float]): Interval parameter (flux-pro only) + max_wait_time (int): Maximum time to wait for result in seconds + check_interval (int): Time between status checks in seconds + + Returns: + Dict: Dictionary containing the image URL and other result information + """ + endpoint = f"v1/{self.name}" + data = { + "prompt": prompt, + "width": width, + "height": height, + "prompt_upsampling": prompt_upsampling, + } + + if steps is not None: + data["steps"] = steps + if seed is not None: + data["seed"] = seed + if guidance is not None: + data["guidance"] = guidance + if safety_tolerance is not None: + data["safety_tolerance"] = safety_tolerance + if interval is not None and self.name == "flux-pro": + data["interval"] = interval + + response = self._send_request(endpoint, data) + task_id = response["id"] + + start_time = time.time() + while time.time() - start_time < max_wait_time: + result = self._get_result(task_id) + if result["status"] == "Ready": + return result["result"]["sample"] + elif result["status"] in [ + "Error", + "Request Moderated", + "Content Moderated", + ]: + raise Exception(f"Task failed with status: {result['status']}") + time.sleep(check_interval) + + raise TimeoutError(f"Image generation timed out after {max_wait_time} seconds") + + async def agenerate_image(self, prompt: str, **kwargs) -> Dict: + """ + Asynchronously generates an image based on the prompt and waits for the result. + + Args: + prompt (str): The text prompt for image generation + **kwargs: Additional arguments passed to generate_image + + Returns: + Dict: Dictionary containing the image URL and other result information + """ + try: + endpoint = f"v1/{self.name}" + data = { + "prompt": prompt, + "width": kwargs.get("width", 1024), + "height": kwargs.get("height", 768), + "prompt_upsampling": kwargs.get("prompt_upsampling", False), + } + + optional_params = [ + "steps", + "seed", + "guidance", + "safety_tolerance", + ] + for param in optional_params: + if param in kwargs: + data[param] = kwargs[param] + + if "interval" in kwargs and self.name == "flux-pro": + data["interval"] = kwargs["interval"] + + response = await self._async_send_request(endpoint, data) + task_id = response["id"] + + max_wait_time = kwargs.get("max_wait_time", 300) + check_interval = kwargs.get("check_interval", 10) + start_time = time.time() + + while time.time() - start_time < max_wait_time: + result = await self._async_get_result(task_id) + if result["status"] == "Ready": + return result["result"]["sample"] + elif result["status"] in [ + "Error", + "Request Moderated", + "Content Moderated", + ]: + raise Exception(f"Task failed with status: {result['status']}") + await asyncio.sleep(check_interval) + + raise TimeoutError( + f"Image generation timed out after {max_wait_time} seconds" + ) + finally: + await self._close_async_client() + + def batch_generate(self, prompts: List[str], **kwargs) -> List[Dict]: + """ + Generates images for a batch of prompts synchronously. + + Args: + prompts (List[str]): List of text prompts + **kwargs: Additional arguments passed to generate_image + + Returns: + List[Dict]: List of result dictionaries + """ + return [self.generate_image(prompt=prompt, **kwargs) for prompt in prompts] + + async def abatch_generate( + self, prompts: List[str], max_concurrent: int = 5, **kwargs + ) -> List[Dict]: + """ + Asynchronously generates images for a batch of prompts. + + Args: + prompts (List[str]): List of text prompts + max_concurrent (int): Maximum number of concurrent tasks + **kwargs: Additional arguments passed to agenerate_image + + Returns: + List[Dict]: List of result dictionaries + """ + try: + semaphore = asyncio.Semaphore(max_concurrent) + + async def process_prompt(prompt): + async with semaphore: + return await self.agenerate_image(prompt=prompt, **kwargs) + + tasks = [process_prompt(prompt) for prompt in prompts] + return await asyncio.gather(*tasks) + finally: + await self._close_async_client() + + def __del__(self): + """Cleanup method to ensure clients are closed.""" + self._client.close() + if self._async_client is not None and not self._async_client.is_closed: + with contextlib.suppress(Exception): + asyncio.run(self._close_async_client()) diff --git a/pkgs/swarmauri/swarmauri/image_gens/concrete/__init__.py b/pkgs/swarmauri/swarmauri/image_gens/concrete/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/tests/unit/image_gens/BlackForestImgGenModel_unit_test.py b/pkgs/swarmauri/tests/unit/image_gens/BlackForestImgGenModel_unit_test.py new file mode 100644 index 000000000..5fbd06c8a --- /dev/null +++ b/pkgs/swarmauri/tests/unit/image_gens/BlackForestImgGenModel_unit_test.py @@ -0,0 +1,120 @@ +import pytest +import os +from dotenv import load_dotenv +from swarmauri.image_gens.concrete.BlackForestImgGenModel import ( + BlackForestImgGenModel, +) + +from swarmauri.utils.timeout_wrapper import timeout + +load_dotenv() + +API_KEY = os.getenv("BLACKFOREST_API_KEY") + + +@pytest.fixture(scope="module") +def blackforest_imggen_model(): + if not API_KEY: + pytest.skip("Skipping due to environment variable not set") + model = BlackForestImgGenModel(api_key=API_KEY) + return model + + +def get_allowed_models(): + if not API_KEY: + return [] + model = BlackForestImgGenModel(api_key=API_KEY) + return model.allowed_models + + +@timeout(5) +@pytest.mark.unit +def test_model_resource(blackforest_imggen_model): + assert blackforest_imggen_model.resource == "ImageGen" + + +@timeout(5) +@pytest.mark.unit +def test_model_type(blackforest_imggen_model): + assert blackforest_imggen_model.type == "BlackForestImgGenModel" + + +@timeout(5) +@pytest.mark.unit +def test_serialization(blackforest_imggen_model): + assert ( + blackforest_imggen_model.id + == BlackForestImgGenModel.model_validate_json( + blackforest_imggen_model.model_dump_json() + ).id + ) + + +@timeout(5) +@pytest.mark.unit +def test_default_model_name(blackforest_imggen_model): + assert blackforest_imggen_model.name == "flux-pro" + + +@timeout(5) +@pytest.mark.parametrize("model_name", get_allowed_models()) +@pytest.mark.unit +def test_generate_image(blackforest_imggen_model, model_name): + model = blackforest_imggen_model + model.name = model_name + + prompt = "A cute dog playing in a park" + image_url = model.generate_image(prompt=prompt) + + assert isinstance(image_url, str) + assert image_url.startswith("http") + + +@timeout(5) +@pytest.mark.asyncio +@pytest.mark.parametrize("model_name", get_allowed_models()) +@pytest.mark.unit +async def test_agenerate_image(blackforest_imggen_model, model_name): + model = blackforest_imggen_model + model.name = model_name + + prompt = "A mountain with snow and a river" + image_url = await model.agenerate_image(prompt=prompt) + + assert isinstance(image_url, str) + assert image_url.startswith("http") + + +@timeout(5) +@pytest.mark.unit +def test_batch_generate(blackforest_imggen_model): + prompts = [ + "A futuristic city skyline", + "A tropical beach at sunset", + "A cup of coffee on a desk", + ] + + image_urls = blackforest_imggen_model.batch_generate(prompts=prompts) + + assert len(image_urls) == len(prompts) + for url in image_urls: + assert isinstance(url, str) + assert url.startswith("http") + + +@timeout(5) +@pytest.mark.asyncio +@pytest.mark.unit +async def test_abatch_generate(blackforest_imggen_model): + prompts = [ + "A space station in orbit", + "A lion resting in the savannah", + "A rainy day in a city", + ] + + image_urls = await blackforest_imggen_model.abatch_generate(prompts=prompts) + + assert len(image_urls) == len(prompts) + for url in image_urls: + assert isinstance(url, str) + assert url.startswith("http") From ad2049f212e8a8e989029426900ad56923a76e11 Mon Sep 17 00:00:00 2001 From: 3rdSon Date: Tue, 19 Nov 2024 15:53:42 +0100 Subject: [PATCH 002/168] swarm - implemented hyperbolic model --- .../llms/concrete/HyperbolicModel.py | 427 ++++++++++++++++++ .../unit/llms/HyperbolicModel_unit_test.py | 218 +++++++++ 2 files changed, 645 insertions(+) create mode 100644 pkgs/swarmauri/swarmauri/llms/concrete/HyperbolicModel.py create mode 100644 pkgs/swarmauri/tests/unit/llms/HyperbolicModel_unit_test.py diff --git a/pkgs/swarmauri/swarmauri/llms/concrete/HyperbolicModel.py b/pkgs/swarmauri/swarmauri/llms/concrete/HyperbolicModel.py new file mode 100644 index 000000000..2b3677677 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/llms/concrete/HyperbolicModel.py @@ -0,0 +1,427 @@ +import asyncio +import json +from pydantic import PrivateAttr +import httpx +from swarmauri.utils.retry_decorator import retry_on_status_codes +from swarmauri.utils.duration_manager import DurationManager +from swarmauri.conversations.concrete.Conversation import Conversation +from typing import List, Optional, Dict, Literal, Any, AsyncGenerator, Generator + +from swarmauri_core.typing import SubclassUnion +from swarmauri.messages.base.MessageBase import MessageBase +from swarmauri.messages.concrete.AgentMessage import AgentMessage +from swarmauri.llms.base.LLMBase import LLMBase + +from swarmauri.messages.concrete.AgentMessage import UsageData + + +class HyperbolicModel(LLMBase): + """ + HyperbolicModel class for interacting with the Hyperbolic AI language models API. + + Attributes: + api_key (str): API key for authenticating requests to the Hyperbolic API. + allowed_models (List[str]): List of allowed model names that can be used. + name (str): The default model name to use for predictions. + type (Literal["HyperbolicModel"]): The type identifier for this class. + + Link to Allowed Models: https://app.hyperbolic.xyz/models + Link to API KEYS: https://app.hyperbolic.xyz/settings + """ + + api_key: str + allowed_models: List[str] = [ + "Qwen/Qwen2.5-Coder-32B-Instruct", + "meta-llama/Llama-3.2-3B-Instruct", + "Qwen/Qwen2.5-72B-Instruct", + "deepseek-ai/DeepSeek-V2.5", + "meta-llama/Meta-Llama-3-70B-Instruct", + "NousResearch/Hermes-3-Llama-3.1-70B", + "meta-llama/Meta-Llama-3.1-70B-Instruct", + "meta-llama/Meta-Llama-3.1-8B-Instruct", + ] + name: str = "meta-llama/Meta-Llama-3.1-8B-Instruct" + type: Literal["HyperbolicModel"] = "HyperbolicModel" + _BASE_URL: str = PrivateAttr( + default="https://api.hyperbolic.xyz/v1/chat/completions" + ) + _headers: Dict[str, str] = PrivateAttr(default=None) + + def __init__(self, **data) -> None: + """ + Initialize the HyperbolicModel class with the provided data. + + Args: + **data: Arbitrary keyword arguments containing initialization data. + """ + super().__init__(**data) + self._headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + "Accept": "application/json", + } + + def _format_messages( + self, + messages: List[SubclassUnion[MessageBase]], + ) -> List[Dict[str, Any]]: + """ + Formats conversation messages into the structure expected by the API. + + Args: + messages (List[MessageBase]): List of message objects from the conversation history. + + Returns: + List[Dict[str, Any]]: List of formatted message dictionaries. + """ + formatted_messages = [] + for message in messages: + formatted_message = message.model_dump( + include=["content", "role", "name"], exclude_none=True + ) + + if isinstance(formatted_message["content"], list): + formatted_message["content"] = [ + {"type": item["type"], **item} + for item in formatted_message["content"] + ] + + formatted_messages.append(formatted_message) + return formatted_messages + + def _prepare_usage_data( + self, + usage_data, + prompt_time: float = 0.0, + completion_time: float = 0.0, + ) -> UsageData: + """ + Prepare usage data by combining token counts and timing information. + + Args: + usage_data: Raw usage data containing token counts. + prompt_time (float): Time taken for prompt processing. + completion_time (float): Time taken for response completion. + + Returns: + UsageData: Processed usage data. + """ + total_time = prompt_time + completion_time + + # Filter usage data for relevant keys + filtered_usage_data = { + key: value + for key, value in usage_data.items() + if key + not in { + "prompt_tokens", + "completion_tokens", + "total_tokens", + "prompt_time", + "completion_time", + "total_time", + } + } + + usage = UsageData( + prompt_tokens=usage_data.get("prompt_tokens", 0), + completion_tokens=usage_data.get("completion_tokens", 0), + total_tokens=usage_data.get("total_tokens", 0), + prompt_time=prompt_time, + completion_time=completion_time, + total_time=total_time, + **filtered_usage_data, + ) + + return usage + + @retry_on_status_codes((429, 529), max_retries=1) + def predict( + self, + conversation: Conversation, + temperature: float = 0.7, + max_tokens: Optional[int] = None, + top_p: float = 1.0, + top_k: int = -1, + enable_json: bool = False, + stop: Optional[List[str]] = None, + ) -> Conversation: + """ + Generates a response from the model based on the given conversation. + + Args: + conversation (Conversation): Conversation object with message history. + temperature (float): Sampling temperature for response diversity. + max_tokens (Optional[int]): Maximum tokens for the model's response. + top_p (float): Cumulative probability for nucleus sampling. + top_k (int): Maximum number of tokens to consider at each step. + enable_json (bool): Whether to format the response as JSON. + stop (Optional[List[str]]): List of stop sequences for response termination. + + Returns: + Conversation: Updated conversation with the model's response. + """ + formatted_messages = self._format_messages(conversation.history) + payload = { + "model": self.name, + "messages": formatted_messages, + "temperature": temperature, + "top_p": top_p, + "top_k": top_k, + "stream": False, + } + + if max_tokens is not None: + payload["max_tokens"] = max_tokens + if stop is not None: + payload["stop"] = stop + + with DurationManager() as promt_timer: + with httpx.Client(timeout=30) as client: + response = client.post( + self._BASE_URL, headers=self._headers, json=payload + ) + response.raise_for_status() + + response_data = response.json() + message_content = response_data["choices"][0]["message"]["content"] + usage_data = response_data.get("usage", {}) + + usage = self._prepare_usage_data(usage_data, promt_timer.duration) + conversation.add_message(AgentMessage(content=message_content, usage=usage)) + return conversation + + @retry_on_status_codes((429, 529), max_retries=1) + async def apredict( + self, + conversation: Conversation, + temperature: float = 0.7, + max_tokens: Optional[int] = None, + top_p: float = 1.0, + top_k: int = -1, + enable_json: bool = False, + stop: Optional[List[str]] = None, + ) -> Conversation: + """ + Async method to generate a response from the model based on the given conversation. + + Args are same as predict method. + """ + formatted_messages = self._format_messages(conversation.history) + payload = { + "model": self.name, + "messages": formatted_messages, + "temperature": temperature, + "top_p": top_p, + "top_k": top_k, + "stream": False, + } + + if max_tokens is not None: + payload["max_tokens"] = max_tokens + if stop is not None: + payload["stop"] = stop + + with DurationManager() as promt_timer: + async with httpx.AsyncClient(timeout=30) as client: + response = await client.post( + self._BASE_URL, headers=self._headers, json=payload + ) + response.raise_for_status() + + response_data = response.json() + message_content = response_data["choices"][0]["message"]["content"] + usage_data = response_data.get("usage", {}) + + usage = self._prepare_usage_data(usage_data, promt_timer.duration) + conversation.add_message(AgentMessage(content=message_content, usage=usage)) + return conversation + + @retry_on_status_codes((429, 529), max_retries=1) + def stream( + self, + conversation: Conversation, + temperature: float = 0.7, + max_tokens: Optional[int] = None, + top_p: float = 1.0, + top_k: int = -1, + enable_json: bool = False, + stop: Optional[List[str]] = None, + ) -> Generator[str, None, None]: + """ + Streams response text from the model in real-time. + + Args are same as predict method. + + Yields: + str: Partial response content from the model. + """ + formatted_messages = self._format_messages(conversation.history) + payload = { + "model": self.name, + "messages": formatted_messages, + "temperature": temperature, + "top_p": top_p, + "top_k": top_k, + "stream": True, + } + + if max_tokens is not None: + payload["max_tokens"] = max_tokens + if stop is not None: + payload["stop"] = stop + + with DurationManager() as promt_timer: + with httpx.Client(timeout=30) as client: + response = client.post( + self._BASE_URL, headers=self._headers, json=payload + ) + response.raise_for_status() + + message_content = "" + usage_data = {} + with DurationManager() as completion_timer: + for line in response.iter_lines(): + json_str = line.replace("data: ", "") + try: + if json_str: + chunk = json.loads(json_str) + if chunk["choices"] and chunk["choices"][0]["delta"]: + delta = chunk["choices"][0]["delta"]["content"] + message_content += delta + yield delta + if "usage" in chunk and chunk["usage"] is not None: + usage_data = chunk["usage"] + except json.JSONDecodeError: + pass + + usage = self._prepare_usage_data( + usage_data, promt_timer.duration, completion_timer.duration + ) + conversation.add_message(AgentMessage(content=message_content, usage=usage)) + + @retry_on_status_codes((429, 529), max_retries=1) + async def astream( + self, + conversation: Conversation, + temperature: float = 0.7, + max_tokens: Optional[int] = None, + top_p: float = 1.0, + top_k: int = -1, + enable_json: bool = False, + stop: Optional[List[str]] = None, + ) -> AsyncGenerator[str, None]: + """ + Async generator that streams response text from the model in real-time. + + Args are same as predict method. + + Yields: + str: Partial response content from the model. + """ + formatted_messages = self._format_messages(conversation.history) + payload = { + "model": self.name, + "messages": formatted_messages, + "temperature": temperature, + "top_p": top_p, + "top_k": top_k, + "stream": True, + } + + if max_tokens is not None: + payload["max_tokens"] = max_tokens + if stop is not None: + payload["stop"] = stop + + with DurationManager() as promt_timer: + async with httpx.AsyncClient(timeout=30) as client: + response = await client.post( + self._BASE_URL, headers=self._headers, json=payload + ) + response.raise_for_status() + + message_content = "" + usage_data = {} + with DurationManager() as completion_timer: + async for line in response.aiter_lines(): + json_str = line.replace("data: ", "") + try: + if json_str: + chunk = json.loads(json_str) + if chunk["choices"] and chunk["choices"][0]["delta"]: + delta = chunk["choices"][0]["delta"]["content"] + message_content += delta + yield delta + if "usage" in chunk and chunk["usage"] is not None: + usage_data = chunk["usage"] + except json.JSONDecodeError: + pass + + usage = self._prepare_usage_data( + usage_data, promt_timer.duration, completion_timer.duration + ) + conversation.add_message(AgentMessage(content=message_content, usage=usage)) + + def batch( + self, + conversations: List[Conversation], + temperature: float = 0.7, + max_tokens: Optional[int] = None, + top_p: float = 1.0, + top_k: int = -1, + enable_json: bool = False, + stop: Optional[List[str]] = None, + ) -> List[Conversation]: + """ + Processes a batch of conversations and generates responses for each sequentially. + + Args are same as predict method. + """ + results = [] + for conversation in conversations: + result_conversation = self.predict( + conversation, + temperature=temperature, + max_tokens=max_tokens, + top_p=top_p, + top_k=top_k, + enable_json=enable_json, + stop=stop, + ) + results.append(result_conversation) + return results + + async def abatch( + self, + conversations: List[Conversation], + temperature: float = 0.7, + max_tokens: Optional[int] = None, + top_p: float = 1.0, + top_k: int = -1, + enable_json: bool = False, + stop: Optional[List[str]] = None, + max_concurrent=5, + ) -> List[Conversation]: + """ + Async method for processing a batch of conversations concurrently. + + Args are same as predict method, with additional arg: + max_concurrent (int): Maximum number of concurrent requests. + """ + semaphore = asyncio.Semaphore(max_concurrent) + + async def process_conversation(conv: Conversation) -> Conversation: + async with semaphore: + return await self.apredict( + conv, + temperature=temperature, + max_tokens=max_tokens, + top_p=top_p, + top_k=top_k, + enable_json=enable_json, + stop=stop, + ) + + tasks = [process_conversation(conv) for conv in conversations] + return await asyncio.gather(*tasks) diff --git a/pkgs/swarmauri/tests/unit/llms/HyperbolicModel_unit_test.py b/pkgs/swarmauri/tests/unit/llms/HyperbolicModel_unit_test.py new file mode 100644 index 000000000..6a4bd9652 --- /dev/null +++ b/pkgs/swarmauri/tests/unit/llms/HyperbolicModel_unit_test.py @@ -0,0 +1,218 @@ +import logging +import pytest +import os + +from swarmauri.llms.concrete.HyperbolicModel import HyperbolicModel as LLM +from swarmauri.conversations.concrete.Conversation import Conversation + +from swarmauri.messages.concrete.HumanMessage import HumanMessage +from swarmauri.messages.concrete.SystemMessage import SystemMessage + +from swarmauri.messages.concrete.AgentMessage import UsageData + +from swarmauri.utils.timeout_wrapper import timeout + +from dotenv import load_dotenv + +load_dotenv() + +API_KEY = os.getenv("HYPERBOLIC_API_KEY") + + +@pytest.fixture(scope="module") +def hyperbolic_model(): + if not API_KEY: + pytest.skip("Skipping due to environment variable not set") + llm = LLM(api_key=API_KEY) + return llm + + +def get_allowed_models(): + if not API_KEY: + return [] + llm = LLM(api_key=API_KEY) + return llm.allowed_models + + +@timeout(5) +@pytest.mark.unit +def test_ubc_resource(hyperbolic_model): + assert hyperbolic_model.resource == "LLM" + + +@timeout(5) +@pytest.mark.unit +def test_ubc_type(hyperbolic_model): + assert hyperbolic_model.type == "HyperbolicModel" + + +@timeout(5) +@pytest.mark.unit +def test_serialization(hyperbolic_model): + assert ( + hyperbolic_model.id + == LLM.model_validate_json(hyperbolic_model.model_dump_json()).id + ) + + +@timeout(5) +@pytest.mark.unit +def test_default_name(hyperbolic_model): + assert hyperbolic_model.name == "meta-llama/Meta-Llama-3.1-8B-Instruct" + + +@timeout(5) +@pytest.mark.parametrize("model_name", get_allowed_models()) +@pytest.mark.unit +def test_no_system_context(hyperbolic_model, model_name): + model = hyperbolic_model + model.name = model_name + conversation = Conversation() + + input_data = "Hello" + human_message = HumanMessage(content=input_data) + conversation.add_message(human_message) + + model.predict(conversation=conversation) + prediction = conversation.get_last().content + usage_data = conversation.get_last().usage + + logging.info(usage_data) + + assert type(prediction) is str + assert isinstance(usage_data, UsageData) + + +@timeout(5) +@pytest.mark.parametrize("model_name", get_allowed_models()) +@pytest.mark.unit +def test_preamble_system_context(hyperbolic_model, model_name): + model = hyperbolic_model + model.name = model_name + conversation = Conversation() + + system_context = 'You only respond with the following phrase, "Jeff"' + human_message = SystemMessage(content=system_context) + conversation.add_message(human_message) + + input_data = "Hi" + human_message = HumanMessage(content=input_data) + conversation.add_message(human_message) + + model.predict(conversation=conversation) + prediction = conversation.get_last().content + usage_data = conversation.get_last().usage + + logging.info(usage_data) + + assert type(prediction) is str + assert "Jeff" in prediction + assert isinstance(usage_data, UsageData) + + +@timeout(5) +@pytest.mark.parametrize("model_name", get_allowed_models()) +@pytest.mark.unit +def test_stream(hyperbolic_model, model_name): + model = hyperbolic_model + model.name = model_name + conversation = Conversation() + + input_data = "Write a short story about a cat." + human_message = HumanMessage(content=input_data) + conversation.add_message(human_message) + + collected_tokens = [] + for token in model.stream(conversation=conversation): + logging.info(token) + assert isinstance(token, str) + collected_tokens.append(token) + + full_response = "".join(collected_tokens) + assert len(full_response) > 0 + assert conversation.get_last().content == full_response + assert isinstance(conversation.get_last().usage, UsageData) + + +@timeout(5) +@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.parametrize("model_name", get_allowed_models()) +@pytest.mark.unit +async def test_apredict(hyperbolic_model, model_name): + model = hyperbolic_model + model.name = model_name + conversation = Conversation() + + input_data = "Hello" + human_message = HumanMessage(content=input_data) + conversation.add_message(human_message) + + result = await model.apredict(conversation=conversation) + prediction = result.get_last().content + assert isinstance(prediction, str) + assert isinstance(conversation.get_last().usage, UsageData) + + +@timeout(5) +@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.parametrize("model_name", get_allowed_models()) +@pytest.mark.unit +async def test_astream(hyperbolic_model, model_name): + model = hyperbolic_model + model.name = model_name + conversation = Conversation() + + input_data = "Write a short story about a dog." + human_message = HumanMessage(content=input_data) + conversation.add_message(human_message) + + collected_tokens = [] + async for token in model.astream(conversation=conversation): + assert isinstance(token, str) + collected_tokens.append(token) + + full_response = "".join(collected_tokens) + assert len(full_response) > 0 + assert conversation.get_last().content == full_response + assert isinstance(conversation.get_last().usage, UsageData) + + +@timeout(5) +@pytest.mark.parametrize("model_name", get_allowed_models()) +@pytest.mark.unit +def test_batch(hyperbolic_model, model_name): + model = hyperbolic_model + model.name = model_name + + conversations = [] + for prompt in ["Hello", "Hi there", "Good morning"]: + conv = Conversation() + conv.add_message(HumanMessage(content=prompt)) + conversations.append(conv) + + results = model.batch(conversations=conversations) + assert len(results) == len(conversations) + for result in results: + assert isinstance(result.get_last().content, str) + assert isinstance(result.get_last().usage, UsageData) + + +@timeout(5) +@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.parametrize("model_name", get_allowed_models()) +@pytest.mark.unit +async def test_abatch(hyperbolic_model, model_name): + model = hyperbolic_model + model.name = model_name + + conversations = [] + for prompt in ["Hello", "Hi there", "Good morning"]: + conv = Conversation() + conv.add_message(HumanMessage(content=prompt)) + conversations.append(conv) + + results = await model.abatch(conversations=conversations) + assert len(results) == len(conversations) + for result in results: + assert isinstance(result.get_last().content, str) + assert isinstance(result.get_last().usage, UsageData) From a3058b6f18cf289aa49858fd1770c59d7085672d Mon Sep 17 00:00:00 2001 From: 3rdSon Date: Wed, 20 Nov 2024 09:18:40 +0100 Subject: [PATCH 003/168] swarm - implemented hyperbolicImgGen --- .../llms/concrete/HyperbolicImgGenModel.py | 210 ++++++++++++++++++ .../llms/HyperbolicImgGenModel_unit_test.py | 118 ++++++++++ 2 files changed, 328 insertions(+) create mode 100644 pkgs/swarmauri/swarmauri/llms/concrete/HyperbolicImgGenModel.py create mode 100644 pkgs/swarmauri/tests/unit/llms/HyperbolicImgGenModel_unit_test.py diff --git a/pkgs/swarmauri/swarmauri/llms/concrete/HyperbolicImgGenModel.py b/pkgs/swarmauri/swarmauri/llms/concrete/HyperbolicImgGenModel.py new file mode 100644 index 000000000..72d6e8267 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/llms/concrete/HyperbolicImgGenModel.py @@ -0,0 +1,210 @@ +import httpx +from typing import List, Literal, Optional +from pydantic import PrivateAttr +from swarmauri.utils.retry_decorator import retry_on_status_codes +from swarmauri.llms.base.LLMBase import LLMBase +import asyncio +import contextlib + + +class HyperbolicImgGenModel(LLMBase): + """ + A model class for generating images from text prompts using Hyperbolic's image generation API. + + Attributes: + api_key (str): The API key for authenticating with the Hyperbolic API. + allowed_models (List[str]): A list of available models for image generation. + asyncio (ClassVar): The asyncio module for handling asynchronous operations. + name (str): The name of the model to be used for image generation. + type (Literal["HyperbolicImgGenModel"]): The type identifier for the model class. + height (int): Height of the generated image. + width (int): Width of the generated image. + steps (int): Number of inference steps. + cfg_scale (float): Classifier-free guidance scale. + enable_refiner (bool): Whether to enable the refiner model. + backend (str): Computational backend for the model. + + Link to Allowed Models: https://app.hyperbolic.xyz/models + Link to API KEYS: https://app.hyperbolic.xyz/settings + """ + + _BASE_URL: str = PrivateAttr("https://api.hyperbolic.xyz/v1/image/generation") + _client: httpx.Client = PrivateAttr() + _async_client: httpx.AsyncClient = PrivateAttr(default=None) + + api_key: str + allowed_models: List[str] = [ + "SDXL1.0-base", + "SD1.5", + "SSD", + "SD2", + "SDXL-turbo", + ] + + name: str = "SDXL1.0-base" # Default model + type: Literal["HyperbolicImgGenModel"] = "HyperbolicImgGenModel" + + # Additional configuration parameters + height: int = 1024 + width: int = 1024 + steps: int = 30 + cfg_scale: float = 5.0 + enable_refiner: bool = False + backend: str = "auto" + + def __init__(self, **data): + """ + Initializes the HyperbolicImgGenModel instance. + + This constructor sets up HTTP clients for both synchronous and asynchronous + operations and configures request headers with the provided API key. + + Args: + **data: Keyword arguments for model initialization. + """ + super().__init__(**data) + self._headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {self.api_key}", + } + self._client = httpx.Client(headers=self._headers, timeout=30) + + async def _get_async_client(self) -> httpx.AsyncClient: + """ + Gets or creates an async client instance. + """ + if self._async_client is None or self._async_client.is_closed: + self._async_client = httpx.AsyncClient(headers=self._headers, timeout=30) + return self._async_client + + async def _close_async_client(self): + """ + Closes the async client if it exists and is open. + """ + if self._async_client is not None and not self._async_client.is_closed: + await self._async_client.aclose() + self._async_client = None + + def _create_request_payload(self, prompt: str) -> dict: + """ + Creates the payload for the image generation request. + """ + return { + "model_name": self.name, + "prompt": prompt, + "height": self.height, + "width": self.width, + "steps": self.steps, + "cfg_scale": self.cfg_scale, + "enable_refiner": self.enable_refiner, + "backend": self.backend, + } + + @retry_on_status_codes((429, 529), max_retries=1) + def _send_request(self, prompt: str) -> dict: + """ + Sends a synchronous request to the Hyperbolic API for image generation. + + Args: + prompt (str): The text prompt used for generating the image. + + Returns: + dict: The response data from the API. + """ + payload = self._create_request_payload(prompt) + response = self._client.post(self._BASE_URL, json=payload) + response.raise_for_status() + return response.json() + + @retry_on_status_codes((429, 529), max_retries=1) + async def _async_send_request(self, prompt: str) -> dict: + """ + Sends an asynchronous request to the Hyperbolic API for image generation. + + Args: + prompt (str): The text prompt used for generating the image. + + Returns: + dict: The response data from the API. + """ + client = await self._get_async_client() + payload = self._create_request_payload(prompt) + response = await client.post(self._BASE_URL, json=payload) + response.raise_for_status() + return response.json() + + def generate_image_base64(self, prompt: str) -> str: + """ + Generates an image synchronously based on the provided prompt and returns it as a base64-encoded string. + + Args: + prompt (str): The text prompt used for generating the image. + + Returns: + str: The base64-encoded representation of the generated image. + """ + response_data = self._send_request(prompt) + return response_data["images"][0]["image"] + + async def agenerate_image_base64(self, prompt: str) -> str: + """ + Generates an image asynchronously based on the provided prompt and returns it as a base64-encoded string. + + Args: + prompt (str): The text prompt used for generating the image. + + Returns: + str: The base64-encoded representation of the generated image. + """ + try: + response_data = await self._async_send_request(prompt) + return response_data["images"][0]["image"] + finally: + await self._close_async_client() + + def batch_base64(self, prompts: List[str]) -> List[str]: + """ + Generates images for a batch of prompts synchronously and returns them as a list of base64-encoded strings. + + Args: + prompts (List[str]): A list of text prompts for image generation. + + Returns: + List[str]: A list of base64-encoded representations of the generated images. + """ + return [self.generate_image_base64(prompt) for prompt in prompts] + + async def abatch_base64( + self, prompts: List[str], max_concurrent: int = 5 + ) -> List[str]: + """ + Generates images for a batch of prompts asynchronously and returns them as a list of base64-encoded strings. + + Args: + prompts (List[str]): A list of text prompts for image generation. + max_concurrent (int): The maximum number of concurrent tasks. + + Returns: + List[str]: A list of base64-encoded representations of the generated images. + """ + try: + semaphore = asyncio.Semaphore(max_concurrent) + + async def process_prompt(prompt): + async with semaphore: + response_data = await self._async_send_request(prompt) + return response_data["images"][0]["image"] + + tasks = [process_prompt(prompt) for prompt in prompts] + return await asyncio.gather(*tasks) + finally: + await self._close_async_client() + + def __del__(self): + """ + Cleanup method to ensure clients are closed. + """ + self._client.close() + if self._async_client is not None and not self._async_client.is_closed: + with contextlib.suppress(Exception): + asyncio.run(self._close_async_client()) diff --git a/pkgs/swarmauri/tests/unit/llms/HyperbolicImgGenModel_unit_test.py b/pkgs/swarmauri/tests/unit/llms/HyperbolicImgGenModel_unit_test.py new file mode 100644 index 000000000..1ad1b8d16 --- /dev/null +++ b/pkgs/swarmauri/tests/unit/llms/HyperbolicImgGenModel_unit_test.py @@ -0,0 +1,118 @@ +import pytest +import os +from swarmauri.llms.concrete.HyperbolicImgGenModel import HyperbolicImgGenModel +from dotenv import load_dotenv + +from swarmauri.utils.timeout_wrapper import timeout + +load_dotenv() + +API_KEY = os.getenv("HYPERBOLIC_API_KEY") + + +@pytest.fixture(scope="module") +def hyperbolic_imggen_model(): + if not API_KEY: + pytest.skip("Skipping due to environment variable not set") + model = HyperbolicImgGenModel(api_key=API_KEY) + return model + + +def get_allowed_models(): + if not API_KEY: + return [] + model = HyperbolicImgGenModel(api_key=API_KEY) + return model.allowed_models + + +@timeout(5) +@pytest.mark.unit +def test_ubc_resource(hyperbolic_imggen_model): + assert hyperbolic_imggen_model.resource == "LLM" + + +@timeout(5) +@pytest.mark.unit +def test_ubc_type(hyperbolic_imggen_model): + assert hyperbolic_imggen_model.type == "HyperbolicImgGenModel" + + +@timeout(5) +@pytest.mark.unit +def test_serialization(hyperbolic_imggen_model): + assert ( + hyperbolic_imggen_model.id + == HyperbolicImgGenModel.model_validate_json( + hyperbolic_imggen_model.model_dump_json() + ).id + ) + + +@timeout(5) +@pytest.mark.unit +def test_default_name(hyperbolic_imggen_model): + assert hyperbolic_imggen_model.name == "SDXL1.0-base" + + +@timeout(5) +@pytest.mark.parametrize("model_name", get_allowed_models()) +@pytest.mark.unit +def test_generate_image_base64(hyperbolic_imggen_model, model_name): + model = hyperbolic_imggen_model + model.name = model_name + + prompt = "A cute cat playing with a ball of yarn" + + image_base64 = model.generate_image_base64(prompt=prompt) + + assert isinstance(image_base64, str) + assert len(image_base64) > 0 + + +@timeout(5) +@pytest.mark.asyncio +@pytest.mark.parametrize("model_name", get_allowed_models()) +@pytest.mark.unit +async def test_agenerate_image_base64(hyperbolic_imggen_model, model_name): + model = hyperbolic_imggen_model + model.name = model_name + + prompt = "A serene landscape with mountains and a lake" + + image_base64 = await model.agenerate_image_base64(prompt=prompt) + + assert isinstance(image_base64, str) + assert len(image_base64) > 0 + + +@timeout(5) +@pytest.mark.unit +def test_batch_base64(hyperbolic_imggen_model): + prompts = [ + "A futuristic city skyline", + "A tropical beach at sunset", + ] + + result_base64_images = hyperbolic_imggen_model.batch_base64(prompts=prompts) + + assert len(result_base64_images) == len(prompts) + for image_base64 in result_base64_images: + assert isinstance(image_base64, str) + assert len(image_base64) > 0 + + +@timeout(5) +@pytest.mark.asyncio +@pytest.mark.unit +async def test_abatch_base64(hyperbolic_imggen_model): + prompts = [ + "An abstract painting with vibrant colors", + "A snowy mountain peak", + ] + + result_base64_images = await hyperbolic_imggen_model.abatch_base64(prompts=prompts) + + assert len(result_base64_images) == len(prompts) + for image_base64 in result_base64_images: + assert isinstance(image_base64, str) + assert len(image_base64) > 0 From c342182644381e3f6a2bc3301a6ad88cbdd705b4 Mon Sep 17 00:00:00 2001 From: michaeldecent2 <111002205+MichaelDecent@users.noreply.github.com> Date: Wed, 20 Nov 2024 10:53:41 +0100 Subject: [PATCH 004/168] comm - modules transfer --- pkgs/community/pyproject.toml | 2 +- .../embeddings/__init__.py | 0 .../embeddings/base/__init__.py | 0 .../embeddings/concrete/Doc2VecEmbedding.py | 0 .../embeddings/concrete/__init__.py | 0 .../concrete/EntityRecognitionParser.py | 0 .../parsers/concrete/TextBlobNounParser.py | 0 .../concrete/TextBlobSentenceParser.py | 0 .../tools/concrete/TextLengthTool.py | 0 .../vector_stores}/Doc2VecVectorStore.py | 3 +-- pkgs/swarmauri/pyproject.toml | 21 ++++++++++--------- .../embeddings/Doc2VecEmbedding_unit_test.py | 2 +- .../parsers/TextBlobNounParser_unit_test.py | 2 +- .../TextBlobSentenceParser_unit_test.py | 2 +- .../tests/unit/tools/TextLength_unit_test.py | 2 +- .../Doc2VecVectorStore_unit_test.py | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) create mode 100644 pkgs/community/swarmauri_community/embeddings/__init__.py create mode 100644 pkgs/community/swarmauri_community/embeddings/base/__init__.py rename pkgs/{swarmauri/swarmauri => community/swarmauri_community}/embeddings/concrete/Doc2VecEmbedding.py (100%) create mode 100644 pkgs/community/swarmauri_community/embeddings/concrete/__init__.py rename pkgs/{swarmauri/swarmauri => community/swarmauri_community}/parsers/concrete/EntityRecognitionParser.py (100%) rename pkgs/{swarmauri/swarmauri => community/swarmauri_community}/parsers/concrete/TextBlobNounParser.py (100%) rename pkgs/{swarmauri/swarmauri => community/swarmauri_community}/parsers/concrete/TextBlobSentenceParser.py (100%) rename pkgs/{swarmauri/swarmauri => community/swarmauri_community}/tools/concrete/TextLengthTool.py (100%) rename pkgs/{swarmauri/swarmauri/vector_stores/concrete => community/swarmauri_community/vector_stores}/Doc2VecVectorStore.py (96%) diff --git a/pkgs/community/pyproject.toml b/pkgs/community/pyproject.toml index a82494195..b9b8d8665 100644 --- a/pkgs/community/pyproject.toml +++ b/pkgs/community/pyproject.toml @@ -32,7 +32,7 @@ pygithub = "*" python-dotenv = "*" qrcode = "*" redis = "^4.0" -scikit-learn="^1.4.2" +#scikit-learn="^1.4.2" swarmauri = "==0.5.2" textstat = "*" transformers = ">=4.45.0" diff --git a/pkgs/community/swarmauri_community/embeddings/__init__.py b/pkgs/community/swarmauri_community/embeddings/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/community/swarmauri_community/embeddings/base/__init__.py b/pkgs/community/swarmauri_community/embeddings/base/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/embeddings/concrete/Doc2VecEmbedding.py b/pkgs/community/swarmauri_community/embeddings/concrete/Doc2VecEmbedding.py similarity index 100% rename from pkgs/swarmauri/swarmauri/embeddings/concrete/Doc2VecEmbedding.py rename to pkgs/community/swarmauri_community/embeddings/concrete/Doc2VecEmbedding.py diff --git a/pkgs/community/swarmauri_community/embeddings/concrete/__init__.py b/pkgs/community/swarmauri_community/embeddings/concrete/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/parsers/concrete/EntityRecognitionParser.py b/pkgs/community/swarmauri_community/parsers/concrete/EntityRecognitionParser.py similarity index 100% rename from pkgs/swarmauri/swarmauri/parsers/concrete/EntityRecognitionParser.py rename to pkgs/community/swarmauri_community/parsers/concrete/EntityRecognitionParser.py diff --git a/pkgs/swarmauri/swarmauri/parsers/concrete/TextBlobNounParser.py b/pkgs/community/swarmauri_community/parsers/concrete/TextBlobNounParser.py similarity index 100% rename from pkgs/swarmauri/swarmauri/parsers/concrete/TextBlobNounParser.py rename to pkgs/community/swarmauri_community/parsers/concrete/TextBlobNounParser.py diff --git a/pkgs/swarmauri/swarmauri/parsers/concrete/TextBlobSentenceParser.py b/pkgs/community/swarmauri_community/parsers/concrete/TextBlobSentenceParser.py similarity index 100% rename from pkgs/swarmauri/swarmauri/parsers/concrete/TextBlobSentenceParser.py rename to pkgs/community/swarmauri_community/parsers/concrete/TextBlobSentenceParser.py diff --git a/pkgs/swarmauri/swarmauri/tools/concrete/TextLengthTool.py b/pkgs/community/swarmauri_community/tools/concrete/TextLengthTool.py similarity index 100% rename from pkgs/swarmauri/swarmauri/tools/concrete/TextLengthTool.py rename to pkgs/community/swarmauri_community/tools/concrete/TextLengthTool.py diff --git a/pkgs/swarmauri/swarmauri/vector_stores/concrete/Doc2VecVectorStore.py b/pkgs/community/swarmauri_community/vector_stores/Doc2VecVectorStore.py similarity index 96% rename from pkgs/swarmauri/swarmauri/vector_stores/concrete/Doc2VecVectorStore.py rename to pkgs/community/swarmauri_community/vector_stores/Doc2VecVectorStore.py index cc4bace96..8aca2ee07 100644 --- a/pkgs/swarmauri/swarmauri/vector_stores/concrete/Doc2VecVectorStore.py +++ b/pkgs/community/swarmauri_community/vector_stores/Doc2VecVectorStore.py @@ -1,8 +1,7 @@ from typing import List, Union, Literal -from pydantic import PrivateAttr from swarmauri.documents.concrete.Document import Document -from swarmauri.embeddings.concrete.Doc2VecEmbedding import Doc2VecEmbedding +from swarmauri_community.embeddings.concrete.Doc2VecEmbedding import Doc2VecEmbedding from swarmauri.distances.concrete.CosineDistance import CosineDistance from swarmauri.vector_stores.base.VectorStoreBase import VectorStoreBase from swarmauri.vector_stores.base.VectorStoreRetrieveMixin import ( diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index 7ba365c89..fb20c59d9 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -34,14 +34,14 @@ aiohttp = { version = "^3.10.10", optional = true } #fal-client = { version = ">=0.5.0", optional = true } #google-generativeai = { version = "^0.8.3", optional = true } #openai = { version = "^1.52.0", optional = true } -nltk = { version = "^3.9.1", optional = true } -textblob = { version = "^0.18.0", optional = true } +#nltk = { version = "^3.9.1", optional = true } +#textblob = { version = "^0.18.0", optional = true } yake = { version = "==0.4.8", optional = true } beautifulsoup4 = { version = "04.12.3", optional = true } -gensim = { version = "==4.3.3", optional = true } +#gensim = { version = "==4.3.3", optional = true } scipy = { version = ">=1.7.0,<1.14.0", optional = true } -scikit-learn = { version = "^1.4.2", optional = true } -spacy = { version = ">=3.0.0,<=3.8.2", optional = true } +#scikit-learn = { version = "^1.4.2", optional = true } +#spacy = { version = ">=3.0.0,<=3.8.2", optional = true } transformers = { version = "^4.45.0", optional = true } torch = { version = "^2.5.0", optional = true } keras = { version = ">=3.2.0", optional = true } @@ -54,8 +54,8 @@ io = ["aiofiles", "aiohttp"] #llms = ["cohere", "mistralai", "fal-client", "google-generativeai", "openai"] nlp = ["nltk", "textblob", "yake"] nlp_tools = ["beautifulsoup4"] -ml_toolkits = ["gensim", "scipy", "scikit-learn"] -spacy = ["spacy"] +#ml_toolkits = ["gensim", "scipy", "scikit-learn"] +#spacy = ["spacy"] transformers = ["transformers"] torch = ["torch"] tensorflow = ["keras", "tf-keras"] @@ -65,10 +65,11 @@ visualization = ["matplotlib"] full = [ "aiofiles", "aiohttp", #"cohere", "mistralai", "fal-client", "google-generativeai", "openai", - "nltk", "textblob", "yake", + #"nltk", "textblob", + "yake", "beautifulsoup4", - "gensim", "scipy", "scikit-learn", - "spacy", + #"gensim", "scipy", "scikit-learn", + #"spacy", "transformers", "torch", "keras", "tf-keras", diff --git a/pkgs/swarmauri/tests/unit/embeddings/Doc2VecEmbedding_unit_test.py b/pkgs/swarmauri/tests/unit/embeddings/Doc2VecEmbedding_unit_test.py index c0dbb3a37..7f3afc447 100644 --- a/pkgs/swarmauri/tests/unit/embeddings/Doc2VecEmbedding_unit_test.py +++ b/pkgs/swarmauri/tests/unit/embeddings/Doc2VecEmbedding_unit_test.py @@ -1,5 +1,5 @@ import pytest -from swarmauri.embeddings.concrete.Doc2VecEmbedding import Doc2VecEmbedding +from swarmauri_community.embeddings.concrete.Doc2VecEmbedding import Doc2VecEmbedding @pytest.mark.unit def test_ubc_resource(): diff --git a/pkgs/swarmauri/tests/unit/parsers/TextBlobNounParser_unit_test.py b/pkgs/swarmauri/tests/unit/parsers/TextBlobNounParser_unit_test.py index 6aa6bec95..e5f8a550c 100644 --- a/pkgs/swarmauri/tests/unit/parsers/TextBlobNounParser_unit_test.py +++ b/pkgs/swarmauri/tests/unit/parsers/TextBlobNounParser_unit_test.py @@ -1,5 +1,5 @@ import pytest -from swarmauri.parsers.concrete.TextBlobNounParser import TextBlobNounParser as Parser +from swarmauri_community.parsers.concrete.TextBlobNounParser import TextBlobNounParser as Parser def setup_module(module): diff --git a/pkgs/swarmauri/tests/unit/parsers/TextBlobSentenceParser_unit_test.py b/pkgs/swarmauri/tests/unit/parsers/TextBlobSentenceParser_unit_test.py index a84023b2f..36c347906 100644 --- a/pkgs/swarmauri/tests/unit/parsers/TextBlobSentenceParser_unit_test.py +++ b/pkgs/swarmauri/tests/unit/parsers/TextBlobSentenceParser_unit_test.py @@ -1,5 +1,5 @@ import pytest -from swarmauri.parsers.concrete.TextBlobSentenceParser import TextBlobSentenceParser as Parser +from swarmauri_community.parsers.concrete.TextBlobSentenceParser import TextBlobSentenceParser as Parser @pytest.mark.unit def test_ubc_resource(): diff --git a/pkgs/swarmauri/tests/unit/tools/TextLength_unit_test.py b/pkgs/swarmauri/tests/unit/tools/TextLength_unit_test.py index 7bc0c94a1..72d3ff6bc 100644 --- a/pkgs/swarmauri/tests/unit/tools/TextLength_unit_test.py +++ b/pkgs/swarmauri/tests/unit/tools/TextLength_unit_test.py @@ -1,5 +1,5 @@ import pytest -from swarmauri.tools.concrete import TextLengthTool as Tool +from swarmauri_community.tools.concrete import TextLengthTool as Tool @pytest.mark.unit def test_ubc_resource(): diff --git a/pkgs/swarmauri/tests/unit/vector_stores/Doc2VecVectorStore_unit_test.py b/pkgs/swarmauri/tests/unit/vector_stores/Doc2VecVectorStore_unit_test.py index 7afcd7097..01c8eb634 100644 --- a/pkgs/swarmauri/tests/unit/vector_stores/Doc2VecVectorStore_unit_test.py +++ b/pkgs/swarmauri/tests/unit/vector_stores/Doc2VecVectorStore_unit_test.py @@ -1,6 +1,6 @@ import pytest from swarmauri.documents.concrete.Document import Document -from swarmauri.vector_stores.concrete.Doc2VecVectorStore import Doc2VecVectorStore +from swarmauri_community.vector_stores.Doc2VecVectorStore import Doc2VecVectorStore @pytest.mark.unit From 99773087821cb24428da035617f751bf15a1f248 Mon Sep 17 00:00:00 2001 From: 3rdSon Date: Wed, 20 Nov 2024 11:14:50 +0100 Subject: [PATCH 005/168] swarm - implemented hyperbolicaudio --- .../llms/concrete/HyperbolicAudioTTS.py | 146 ++++++++++++++++++ .../tests/static/hyperbolic_test2.mp3 | Bin 0 -> 12429 bytes .../tests/static/hyperbolic_test3.mp3 | Bin 0 -> 13731 bytes .../tests/static/hyperbolic_test_tts.mp3 | Bin 0 -> 10005 bytes .../unit/llms/HyperbolicAudioTTS_unit_test.py | 141 +++++++++++++++++ 5 files changed, 287 insertions(+) create mode 100644 pkgs/swarmauri/swarmauri/llms/concrete/HyperbolicAudioTTS.py create mode 100644 pkgs/swarmauri/tests/static/hyperbolic_test2.mp3 create mode 100644 pkgs/swarmauri/tests/static/hyperbolic_test3.mp3 create mode 100644 pkgs/swarmauri/tests/static/hyperbolic_test_tts.mp3 create mode 100644 pkgs/swarmauri/tests/unit/llms/HyperbolicAudioTTS_unit_test.py diff --git a/pkgs/swarmauri/swarmauri/llms/concrete/HyperbolicAudioTTS.py b/pkgs/swarmauri/swarmauri/llms/concrete/HyperbolicAudioTTS.py new file mode 100644 index 000000000..b6ff4de67 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/llms/concrete/HyperbolicAudioTTS.py @@ -0,0 +1,146 @@ +import base64 +import io +import os +from typing import AsyncIterator, Iterator, List, Literal, Dict, Optional +import httpx +from pydantic import PrivateAttr, model_validator, Field +from swarmauri.utils.retry_decorator import retry_on_status_codes +from swarmauri.llms.base.LLMBase import LLMBase +import asyncio + + +class HyperbolicAudioTTS(LLMBase): + """ + A class to interact with Hyperbolic's Text-to-Speech API, allowing for synchronous + and asynchronous text-to-speech synthesis. + + Attributes: + api_key (str): The API key for accessing Hyperbolic's TTS service. + language (Optional[str]): Language of the text. + speaker (Optional[str]): Specific speaker variant. + speed (Optional[float]): Speech speed control. + + Provider Resource: https://api.hyperbolic.xyz/v1/audio/generation + Link to API KEYS: https://app.hyperbolic.xyz/settings + """ + + api_key: str + + # Supported languages + allowed_languages: List[str] = ["EN", "ES", "FR", "ZH", "JP", "KR"] + + # Supported speakers per language + allowed_speakers: Dict[str, List[str]] = { + "EN": ["EN-US", "EN-BR", "EN-INDIA", "EN-AU"], + "ES": ["ES"], + "FR": ["FR"], + "ZH": ["ZH"], + "JP": ["JP"], + "KR": ["KR"], + } + + # Optional parameters with type hints and validation + language: Optional[str] = None + speaker: Optional[str] = None + speed: Optional[float] = Field(default=1.0, ge=0.1, le=5) + + type: Literal["HyperbolicAudioTTS"] = "HyperbolicAudioTTS" + _BASE_URL: str = PrivateAttr( + default="https://api.hyperbolic.xyz/v1/audio/generation" + ) + _headers: Dict[str, str] = PrivateAttr(default=None) + + def __init__(self, **data): + """ + Initialize the HyperbolicAudioTTS class with the provided data. + """ + super().__init__(**data) + self._headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + } + + def _prepare_payload(self, text: str) -> Dict: + """ + Prepare the payload for the TTS request. + """ + payload = {"text": text} + + # Add optional parameters if they are set + if self.language: + payload["language"] = self.language + if self.speaker: + payload["speaker"] = self.speaker + if self.speed is not None: + payload["speed"] = self.speed + + return payload + + def predict(self, text: str, audio_path: str = "output.mp3") -> str: + """ + Synchronously converts text to speech. + """ + payload = self._prepare_payload(text) + + with httpx.Client(timeout=30) as client: + response = client.post(self._BASE_URL, headers=self._headers, json=payload) + response.raise_for_status() + + # Decode base64 audio + audio_data = base64.b64decode(response.json()["audio"]) + + with open(audio_path, "wb") as audio_file: + audio_file.write(audio_data) + + return os.path.abspath(audio_path) + + async def apredict(self, text: str, audio_path: str = "output.mp3") -> str: + """ + Asynchronously converts text to speech. + """ + payload = self._prepare_payload(text) + + async with httpx.AsyncClient(timeout=30) as client: + response = await client.post( + self._BASE_URL, headers=self._headers, json=payload + ) + response.raise_for_status() + + # Decode base64 audio + audio_data = base64.b64decode(response.json()["audio"]) + + with open(audio_path, "wb") as audio_file: + audio_file.write(audio_data) + + return os.path.abspath(audio_path) + + def batch( + self, + text_path_dict: Dict[str, str], + ) -> List[str]: + """ + Synchronously process multiple text-to-speech requests in batch mode. + """ + return [ + self.predict(text=text, audio_path=path) + for text, path in text_path_dict.items() + ] + + async def abatch( + self, + text_path_dict: Dict[str, str], + max_concurrent=5, + ) -> List[str]: + """ + Asynchronously process multiple text-to-speech requests in batch mode. + """ + semaphore = asyncio.Semaphore(max_concurrent) + + async def process_conversation(text, path) -> str: + async with semaphore: + return await self.apredict(text=text, audio_path=path) + + tasks = [ + process_conversation(text, path) for text, path in text_path_dict.items() + ] + return await asyncio.gather(*tasks) diff --git a/pkgs/swarmauri/tests/static/hyperbolic_test2.mp3 b/pkgs/swarmauri/tests/static/hyperbolic_test2.mp3 new file mode 100644 index 0000000000000000000000000000000000000000..1fabb16fa56f2691a3d9538e81d9d42b97a0ce53 GIT binary patch literal 12429 zcmeI2byOVRw&xoSp2j6Wppl?S;}G24CAhl;4-i6V+zIZ%wQ+X|7A#nRU%>2$;wdzz?)&BPRoZ3}qpR<9hgh9~%TJ#;B+aZ6ukK80c zAZQ#27a#vVAt5CM!ut3z509Xjn6!d|s4*7mbg@n^(2cW&p+Ho1DMfM=d^RK4_i>XT|xFMCvnHL}=A%EH5WZ zh@ki3_F)l8Jaut-Fpt*C5XeBry1&~3g=t_3K7P)}i$x(}vR9ON&t!j9)<4;$oQtNz zVbtY2554#C!0AlSYft!u)&t##eDt0QgV=3cwKuMtpwGEXsmp@1pv1xyaJ@7G6N@7> zNkb7O4E@2cMHz!B3PMVXHH+wzj0rlbx;nJi*A6HH+)i_0{h?YADO)*8pRcW32+GE2 z^A8~g@cMK$OgVeyf;JhDo14bzQ6Cli(Ri#|5r_#5cd=c$6u1HpZlm>EqfwwCyI%-o zC1DN%{bKVvKm{=qyd}yEbL;=g$D5Pm@Ka{V&3Ba2k{7-BL<21k9cw`$bTMBVqE1&x zYXmN0gl5fZGBd|D#5%?FTCSzGuuGV0k_^T&s6hlJ$DgU)=^UU)Ct#34U?7k($ciKZ z1X63LJa#0q-e_*t)>_PV5948mOq*Z06qrS<>WD$iUKw4MW3ut5Mu%u4#O+Jb!`l#?l-o2lEE zKmCN|^06l#n+gSq#DEm)j0-ifR{9!F+yqQtEjDKwgoYY+kFZ}dXhMlgE-{ppBX`TL zGOr28dXJ}dmq**B!IL73e|=t^T)y$F{ncgS9*;CUDghgxIl14~{+=6|oNpj55z9E< zG$3HC&0bU`1s>vpvbTJMiLgpDCRla~!Ew5VNu!Irfx>Hay z+MD8L`}qexji=KX%q$Q6)`OorqEw+okM0+lzxhip67EAu`bvt=s$0FI(^nP(B6b-c zgqZC+>pMphk(O42sg^8%DwxAid@{WnKz>gSKvY99qE(_Ijzvul*T)Ph z0iPFVerAjkqCJWl#wB~0-!VjB%F-fNpCH!j7qG4q;DtB~2^GeCqaCd=My2rWAYO=> z^u*NMvwPGm`wd5_gp}iwa2FVTpAsh%moTN6E4PB2IMSLiftk^s&6}Y)n$}!Y&&-lc z@(_!=L&^PK<=iVVWskB}t{PxmbWYZiZU(EL#J|FYu)R!FG73JmM(MJ^U!?p1vNX&4 ztc@bGw)(?|>qo%U>1|gxt-((rQRRUqUh2qXff=2cm%E6A2_FgmoR_Mj`E`;WgXErL zX05Uk+89p?)(PoYX$Rg3eua-#o%DI=H{um54L*wdsqY)oFlY(#y&}VSu_EaWbkJFt z()JCfW!B$M_t1}jm&Y<O4i1G1?V-h0`$RI4_*ILZK5Zi6Q1SR1*7U#Zb zME)jQc~}KapjmhC>6f@?MJew{G?E;3NAgoz$+$sA&lx|qzugy>G_xlO?lTFZB#}t- zN8z21LG5~S^F@(F%OMF*X;!c`!`InzE_cK*U}C?m#88M(q_SyS$V*RCFr&WkNns&( zQq`O8^TQ(AGC@h1VeEWZl+5Hqq54UR`^&oo3~BxZL|qfJf{%rg!BOKjqua4^qC6mt z3`LC~iJ0-!N4)eJd>0UQglC zb7EtFwyTt}lI$XeWUBpo5iX(+X|?0X2sL7u#C&vL_bx<^e)006wID>#Z)+b5g?&2? zZ8FpYWve7(43q1OR#|;WAB*UT90%*T?LTwT;9XcqVGMrb2wa&DUcd-GD7zz|l@1-% z)d7joB1GnPo4s^~=LWeDJ#KXlMCE&*rYh1c@J}hvy#a{Ot}nB2Oa=|XH~{Eawv*LL zRidHOQ|xOk9cR6x3A|DAVkKvPI_gE@fg)iEcQ`3|UsH}+o!BaUZJHAS7da=Sc^@10 zW6OZTsG3&(**bsqG+!1!-Xb;jHSagJ4F?^)Ecq3zT5j>>i|`=&lD0sv7fz0NwX2lz z0}bT?CaVBoJ=tQb>eQ_yl|s3MchDcoC-3uc!*yUHm0>Te&|1{H^jp(F)C-A?mhSiQ zs?W(4{1QDVtJn8~&1aig0+$+FNZ%80y&YmQ8#&wLTF0?>!2qr_1_2O(BBcX@qMm-C z`am=^b)V(yIG8uM8F2iOMwnF$n*gB5KoWr=2kZ=0BbH2}PaJRq z#x&-wPu~h;=Syd za;7ARy_{c{v%GpXgu=caYV3L6E7!EU72vzU>&OVC|vSa|89_`i5S{ z-%f#77uS{RTPM2`KrNMX!3%>#<(DLyxIFB@&6TLSdu;3V^|*EoA8(y-pigeVp2v~* zJ(F}f3p}jp3CYH`y6xrerX9)61wA><74i{7jV|mYl{7}E97hJ5AgXfXV-@lbg}>Ko z2&&~wJAFS%*q_|Yj%9D$N9_2Zb27uXzlP_$?EU_VlynFs2H+GgnXH2Aa|ob0Lc%i~>Qzy_hMnAF*%eV}!%boha&5Q$*BX zW+m*G_Ku1h0-7ZoF)=bZ?!Kz-LwUO*LgRkuBK&Ub_NPCd5h`hl?iyPqE{I11djn- zUta=~{-#A!q%?R(q5*ZJzumjshhKJuE$8CGK3H?T*cCZSFdsaE-M8_xLr}N}H9h7O z4act7**V;?#uk>}o`F9kPDGqi#pqEDYoRE7WfL4|e$acgaIhtfC9~)ei34>r8CdTW zI6+uHs_k8a6umWszc0#E65NEwRbjoVXm8(1d*=e~56$L*b$iNzQa{WQQlKw>;Epne zOGNlG?J8ASR3v1MkzPgZdPdrlbinUZ>MS`3DdQ}@IB{$=qjjH@R_dWyR#Wvd^M6rT0Yn^Yx zqLvV0)$=W!Qa9fBd#zO8qJYEsvY!@U#cqrQe&Cn5pa8N&^x#5!_ zdEZCeukjOt9~@NwHA51|TOXWBe$ld`RZ^c5ZaWsT!2Bpdju~dw9M9nZm*_W&>KH;v zRa8t?CxI2W?DaLTJF`E~z-I-4x4<6;p(bwZ;sLewI45uCuMU-ER_l*qL$r`V2XGr0 z&s6%xi5~J?cMW+67+t~BUS^q+u9IoamVmKm&mU$C7>(}cxty=c#v2eaI?kJHYn>Dyd6gdaSMK>i2z`Xck z5Wh_{>9QVtH2D}biD}+joWcBZ_Mg^=hZpNJEFAhcJM6-zi6HHqm5#vI9fRnV3W)`y z=9M&0ce121o**$c*dAa-xo}&Xna^h>2U6H`qehZ;p8K+rBrnCOZoX|WF3cOfP;X>f1GRf7j;+_F_`YF?Ohd3>GM_D9`4->ivyk(b-XM)Y?KvvM|uEsgn?rtn{Ye zJnsp!kh5Ot%p^N;Q+1x0zw*qmPX<(={K<|zg>-D{d-C^WmpmbQ zY){&5^-Hk&iSeUYYdV5$2~5ksaGZ##G&4FDRdF^NajMT%4R=iJdsMTv%CK;GY^{3k zUxl7wmfmoO+ZFiJfWUOqQZwXa&~k%(rzg48x(|Y&=CFi^!oI zhK8uyL3{fRlA}?t==~*`j+a!`V!vb&@5J_{55KP}?E(u=U_h+!Mjlm_WqlQMFuB*|SzrLCz zno=}f1DD+mNthH;$s9cjZ1lxgNq5218HmD^T=JqK32m^+;k+IhACD1D0h1*Hwst2) z4?q5pP4vUK*C}vq3gZ@#nFVB>Hh@ibQ39Yp%^wiu1AhII$H*pHWJ>9quHDmf`MxrC zk6~dIlVn8LrdI3w5q7qTPYV8=^ozB+^y- zsn$XKw>Gr}&l@y8u~|YHly^(>IHgjb3lEhmcRtT6Jl94K85AxcK8SXQq|VS3lPFhf zpU#jRLyH)MMMnh_8G)NbPlVmmu8Ch<`5Sy-tH5&d$hYr9>Boyx*>P9VDFdN|)SL=k zLx9gJU-Ym|9V6O9m_tE@Zb4rY>#gR;$;a5*wY|x~qYvD#0@EL-eCgJHieMUW!K+dG zVg@k#r6$a51p<$@ZxTc&*i5L)o9^Bs>mUN1N+Sg0hp0~_i0%+FI-)k?H~SVOn5H7e zjF=GII9`?}cpsB}WjNFr8U~UP)2Dh=-&xM{x grYsu(@6#+GDlR;h6E3F$R@1ID zwfUbh7BJW(l==1a3K7!CIaiAA#(X8s30f^guc`=wQuZd)6gkG z*7Dp;0NqH2?A04vCT7C$*&)xfiCCyVsgJJ6iHL=z&C`%Dx>?fI&qV$?@DWsM{i^d% zvcO!jFmrj;Fl~*2So7)3s2$bj9fVET{X|!*eTqix;S%TWWCA*=`=pGXg-3J#d12G? z?L8e{)i22nsBBnD&W2;Srm;*liAZfyVq>5#S?tU6ADWNdnW?o9Ye40gG|Bd<6M;KNrP*$CO06|(39FhA zk6G9AqL&M9%pZ8Xc5q!k@es3(Bzj3z8WfdaMr}z7&O8FQYAk}G?i7cFpJu#-KK;Ie zPrkxOa2cETt2}#hv3d5}lf;hZ zE0eLo?8(Zd>7*2Uy|Na3Co^E4Vo1JO17X8v!-ntUdb0;fO(72A7Iz&U*c|S<2|PUM zi^_8NjQ!Zw*e#U!elfQlsSwkaRJ7E5h}@(7$Ez)_>i6Gf@{VTvdoF2_JvT2Xe*cCi zPV|`!yvamuuA+g1WAha!e!iO={ZCbW|M>)qMxZQg&z*JU-*mQ{Zw@JbXOIDsYMS*AOjT`ase4kgF z{>8(%P9X*+dvOz{KTB2`D>3zks*G=Gp?3|q%@O4Oq@QiALY#JU)5TGtT7p3;(>Z*- z(DBj3yd_K{3bHx~py)555^jafNFSk`JQI&8Z9!l}1> zNQpA%L|J>%RQsL;&Al0=%BqC7V3Qvf1VeQbH ztsy@_f5|#aUP&tj<(@9u3&)8&2tQ#2?M7Az3iZ+3y1nkuf^TrWo1WH-9FcK`6C%(# z9s1l7Z=?Zls24?agesfwqD&LPEtGU*Dlg?#jGql8@78-wlTUp2Tu8cJE3s9iW=h_u zNSHI$o&G)5HqQ(H-XbbpZHA=iAz8QyV_DC_^C^!uXFSadmyA}6pfWEEZG9#|q9Rf2 z(NWLx>S0(R?QokVZo+rH(5}}p8S*kN_)kYGVo%-Na%^#`LtCuQmca;z^Hk>+Bkzpg z2LnF#RjIF>E<2v(d+SO0YbWOM&0}39Invwsv;q!nvaH%dXx$Q64>*-?XmsGDO6MdR z2)cEJWJmJ!Wsaa-3@u(?Ir%#X$PjY$z7>Mb)1kMLLo|fU{IOr^jR80^Vfu(x&pwIC zF4&ATzWqCAQkqbhUt5qMr&CJq|QyMR=`$k?(k6k3nL)1SM z3w*00M9bI)cp z@S$ERU~5g8+`{KordXZ@Plzk%QhO7=Z&H5k(s%Klzhm&6A*~_av`n5RHz~5Z5Qs);fmsj41rLzuehJ_Q;c7 zU-P`L;77GP7kZ>O9&mXcuJi4a_;oAvmuYL@n3!M?8VRt^b4F9{d>eYoq4xP}k#C#Y=}#g-Bys?nJ(a?&>xyyAg#f zgzmgiAzC$tvBYtxl_tJEEOAy9wADMpTw_v=g?80&@rECwhi?sKx?F&ZjI}V1;P-Jk zpMl9x`HDQ?yJehx;?a|dJuI4(kH=zz0Tpa%7l`?1SFc?Zig_}(t64ZG9>PZ~^CEYfwu_1Q{;TS|Fgasz|qg6Z)YJ*OIowt(YiTB z{5UiI%j9$I)JLxO@D9umf9r4$gmtW1>7Q;t5UFa&w(R=8E86YKJTDr;c& z(S(bn-czCir|S6^Z0qH$TF7ss4RO5~0n#5O5Iv=;WSX58o#HVg!cuRevr>8+%pv9*t$ z8y!&=HWw$se0}hLeqb)2LDM63Fm z8Z9GPQs1O3y~|}jw6E6)J8Y+QP}HdETw7Qr1Hc%Kf((D}pGnPVR84X1GoX$HC=2oz-Z29K5XsiegQU-*)W!}vk5Ha|(T)dbTQ z$Jv_x?*ercO818fWS&?iq`CYJf`J_}?Ok z?liXU-?kS^nh*8@z;Vs@$A#!rkRu-HrlLULCgO^4_1kyXJW{${md-CiD6tkDMnw=> z6dnk%V4$vxBCj<4CI{*lU5uO(wsz5c3>M!nn2`PMdJHsw?oaAMt2D_j0uO%6Z#V5d z6gxp#zYGM9GnrY&pA7E-DLoOW6j#7l3QvyQvPemNb~7qENO-|pFoCAXMU51N7eklDyLT`I5rgymYi$Ef&1djcXL<_tZ4;> zv+G+s&50q4Bd@o1PhN^ORa`Y?1p>WI#m#TJRz$=ZH({vWNkqegA@d_%aWEQiP@_PL zU`|_f$|K>EG>uVjnytFl^rJLrxqD?xi>+(|7Pg+jzGHfuLoHLr#MPO55Ehvq);J#g ztxApF+WGBmg>Mlg__~SWXs$5=ws`rc?b!sls zUBU_kf>9FEI+eEHb?QEdpuk2KIZgJh`JrQX2>x}8Tw!a|5mbi(P0=;ZyWN+bn$|1M zy%iI+<&(!(XC+F5cY5SRLK3t|R1!N45p26#Snr+6<|`WA6l(c?|4fzDGd1je8+wno zH^A01L!oMqt+D1*mz|Ww-&RJGO6;@uC3h4qee%gBPgRF7H6w(Vkz7PIk$;&v82@75 zEq=96(FAz0^DUJ~hDP=bTtpZJHbG1;HPqR;`I z2no4tg)n6ruHEph3iJMUOxGTx;bv%9am|VN!moF6C0BX>x;KZvr%6$h{gf^y#-30I z3i9*++%kxj|1>=Q4#F|)ewdv$f(vICmFq`$O58OH=ml|IFMVhM%rd16WjHQQ*Ra3f zwBg-DjYd1x%yUb%mF~ikx79241Hqc`+eXUOS^dPi7n3Z4xrf8w_fo{+4qS^ow zy6httcB6eT>T*L_)Wf={hno1L(HV+n<6n9dKo!x5i5w`@ex&Pg_GokK4w zG%8a~MZ)(h(nS9DBrI7eBl2r2QkiycLhw&mQ29nV-a1F3R>^P(_kQ9Bp>`Of=xv4>o2T*YIZ3YSZ!{W(;@XE=%;fS)-IJ#w46ADQNB31N+=KVn+D- z*t)bJSGXTj4dvsRL(*F#6eByCu}ZpgWN9C?qjbd&ZXz5Ex~yikz!h@@c)%{Uh<=@3|8H-d+-f zMF&D<7Dffe8E&`edwnXAx0`~nFhTq(M!(draVqcJ`y7Pldm+A^;a*^vT#AH%TzC2L ze`NmelG7j%9SEZ5_*H=bL{kJw&SABDad2t6tNvf59Yj*&cT9TgxK= zxX^ahCj#Y34NvYp{*pB}VPe4YW?B-Mzkt$706 zi1h*=|Iqn6+5h_f-Dq}{;!R9{(WwLvd(`4K5<*k literal 0 HcmV?d00001 diff --git a/pkgs/swarmauri/tests/static/hyperbolic_test3.mp3 b/pkgs/swarmauri/tests/static/hyperbolic_test3.mp3 new file mode 100644 index 0000000000000000000000000000000000000000..f8a7a0a8b5719ae56d3c9cd37bb57e1e0c528fda GIT binary patch literal 13731 zcmeHt1yEe!lJ20v17sir0Wu6ua0w2Bd(gpxyN2KpY)EhmuEE`cLx3P55Hz@k1cJL; z0vXPOySw++zPq)r?yKFZC%3C+&Z#5+_xIQR_vyc@f$O*)(EluYb}wvhuid}>$$&uo z1t2^EA|hfkGAdeH#wSlW`S?U7CFPWq)U@^VjLpp*U0l5U{DMM5V-ge7va<4viz{my z8(Z4jKlSwV4Gau@{W>~6J~=%-J3qhhV`*u5b#;Anb9;Ap|M2km^z{7l^7r*M3b@4* zRgqMc<>lb!;=-){w<(c9>9Ob}L1@E&nMabU@4wyn|K$i==k|cmx(F~}AS4LNi7e2L z1EIgfj;8|tBIj3RxU8!E7cb7HwXgvHM-k;3&Lh5yQ6bKjj7@yzvRtR!ea;>_L)U?G zo%82#?9s1Iz;11&bqZaLE|cq1H}|>*!>`e=oE7i&8;8(cpgK{?fE4-|;r;85A+eZi z=kFI~t-7iKUqcWYW?#Zvn3dKi(ycsf1OU&&F-egS06;x5@VPnc-1oAcKJ(ac+4kU^ z{;L@g|0<^U9s}3yc#=iix6eoHUN0X+M|N9 zsCGN?oL#<9y?eEI>@B==yv=HezYXQR6e>O=>KHgVQM876w5K629XF^BrFJF1l4yLF z{1rStO*Rg*J#fViLgOac33~?`({llBhh8plprcGT(C=B)Jw0mP%X)|XDHETHxom=i zQ5XCn<{rCBAu}xe!@vZ#%yzlo)SGzIX$)wD-}h|+SzWU^K2r^ zc=0iP1%UH={W65uhAyq8&;j6@xaSP5*YX}1GZ+NQ0GTJnfk3fmB<3&2IufgdQ{q1Z zz&ioN(~!4;&rPw7vAG1h&-l>nsh|xUFmv&sVFzgCit020?%J8NSkhw%t4w97Soo$P z;h^9oa;z3_Lf<*LFNzga@E4pnGT*Vl7)TJv6qMg{ixcA?CRItd0y)A=#oQhyV3_|H zh^ZwJZHKD(9W8>kGEFcE#1|{%o@ezplY2NsdmOSYi7}E69lc|ga$G!+bp3FV<-p)M z=q$@ysbl-2+40lj=R2GW5PBL0gzk1Rmj}@u2O~i>o$BVi3f`|Yc&(D0y*^j`w5O?v<&-iKb=#(EG2X5DtD04CCoe!h$9csQGdw1Y0wpoz7i z^|(RDposhil(LLgn54yqHF;V;*Pb%V#mZFSi)n97zrLp8%b$qXuN6&MUs$=1=%(HT zidT+H@YqBiF!$9uEmssiuI~9W-6CJc#V4d`n~ZBp+NE&+N785J+A#0LR8YU44LO$B z0z#=dmTfRxQI46AKTBrm$%Pz5Nto8dvJLlLh<;kK-ayhvoZ%7QrzJxp+Y|3)EuYm* z*gMJXT}bnNY#p|WpARV=S+@cF4XQ03K7fX*f2nGSlem)3XSEabsm&GEYuITMq|tiF&3DC5@MiX=5t}E)gWC~82R#~gWx1% zGI#})dIGCYbE00Rj+)@Yz_#DhhlXSJz7d4eeI2a7xo|g!%-3p^khIX@xrH|aWYXf! zz*w}@h;sjqtm6;;@Qo z72dW-V~`W8XL5Mr9ODc>|rs<4YGLJdRoyaEpfF+|DPmZunOVQr_j$1$W?e zyhgD}mG~7&n@+5aYfKFJOU+XESW!8KJ>f3y!Tw~4eds;OfdkB#6Oy2J2It%)g>`%R z-({mT1sr#T#@&e6IFl?OZkRD-s^%|GU}*Q zK#MmJIKPlfh!PehfTb}A2Wq3$QKyp|C1b%$m1?lDtji-QQ_aE{L_QX49)U#|N%K4_ z(k*mO#@W71zstaTo%V^Qlkib|KX;NpmBr!ZQ4w%fPQ*pPZ$AqFe^?NlQyocyFaH35 zc=hy?)1rBWd8;5fSIz$A_y$2sEO>I}OBN8ipSP#8xJ&74G|~^kTVy;`vJrHN`N(8B zI_x#Us_~h?-_)^pB>Vyq8HHHwLF$%&aj~J^;1zy&3d`;3wjaWU%G6I1ab&Nq9 z$fMCbzvw4??7SwSj+foDZ006dlU=6qW)H#@6J%4_K3MZ2>i6ZrExbajG<=ixkrE9F zb!QRj*1xXaq@DuT>OGf8LKa_#J*e7tH)o}PD525Aog9EJ>x|heZn3IDy(U)#Q->J@Y?}Bzy0Vlo1_=Cg4bZX{t+k0dVRcUukpJe zsR=hYbtmQ?d^Qx?IMY0SNrrrW_@YJAwO5%!{GeTU*z7=*hVNmlB(5*zkAt~)gZe2{ zK_-{}YnBA#!fba&-2xFoS7rAhE@n0`Te0Anq6z(|{^jeG@Ar&yY$_f!6}{~Z0P34M zX6Gx7xR*KTJN1M{4QVqJ*T(2|)LeZ}KNASdUp=$$<65T+59`h!XP7DXq%Op zm@*(6T`8kQYf?YTn_m5O88MTRun%!lS`@s!5LIES)!@&{chfJs1tKppq`HahbC+s3ngOk_ZE2uTx7s>Ngzdu;(u9KI&Pei~naK8oHLjRs%uMTZe^j7oKO3}-$ zvxhLxd9&l?C7>!gYIuj0fL>Rl7{&`6eveE>b=n-*$ifQyD`_CETr-~)U?*ksrx)7E z1o+zUAY;E$Qd*5TTsD>gf~2R`nxiq4*xw-+zKYJr;T^BqM0SoE+J+G6_4$)t@@C{h zOo1cDd;1c=yc2{rWvN`6lHn34N3SnnW{4oiVTPAWy8~?z+9JTsmVr;&T0BKK2`1C_ zY55U;Mwo~9dHTZ~?p!|e+AEH962dSSJ4{KOEcvHLA7fj%7@Lh~5Ds@FTp@|nQ8s(+ zxMma8RqaXSR?0r+<2&b;T$4JCjym~e*vvEW$4gU18X1>}?9)BHkLPjK(`1pQ<8mJ>NRHf-aWkoi zk+Cm1)&s>7j=IAeyuF}xiON>*?XU`DcX}i>lgqE2aPVtp%`$6AP{BAWlFMa4k0HFI z`O1yNV*Z>))M%y1ftTqD5t$00l-jokH-*6EtXTJy$1uaZ^ z>-#GIH+V&A6`dRu1c+TNMk>%dluU+$1XD{ zkDq5i<70yZjf)^QNQ@f3DlhYp0_veZaq*mit;%U-@N2XO5jB?WKkU%!^H zV4h3}gkVVK8a$`I6T^L0i94fiBMHX?9ri>^@><^S zp~8~cpIEoFPRxafhLIW%-VaV|R8?N&M|{b+;H%1|Jd{^XehpJh4#AXtp$*J^+T@CB z2EL~MP}1S@09h#Muy_C;D=nJmJ002{ZrNBd$A^@DAtiGK$p*{IHD3pL#-oK+zhvtb zCSypPn4ZsMv*5y;j-C*#8=N{*N5n2X+x*UaJ%!JM^PXeniD$c^&}`tNrhP+zEj}^8 zL|c_|AVa&}U(E7_&7O>qA?!Iydsstir#N;0nGX$u93;0ciA5lzZCMuR0pjEvPuxb5 zkA}%Gn(U=3ND~tgpcB&~Tfn@yd$fjVWauTz)#T(4_nb;^6MEC0QxKtsm(3njqwM4R zz4jEcBIq{cfUP0!tS6(1zc0ZePuU`|b>dgB@~?;pgu(z3)oixz26u~al@>*ll5N^|5z%p-5uXolV=$kJ!+I}8d=!Uf zIAS&~3M%-uB_pT>$`}5n<0(9fh)fqL@f=fr_ZYvlPBdFeFxuw>FDJ8okHy@|uMpD6 zQw6(Ovex?rT;FaszB@V&MO^v?UcVbLU7nGxP-So~)i98ZDAdyok7c~y^B{KO&*?}7 z0bwi!-PNhd<*2_??6wtnGVEa^x(0^Pc{zylVnz{F`e8rrE#xnr4Z8INlfAT><chh>Vuz3`xI4UL>tOTZlVZ8ih(zWruxRW=Q9gKH1QNt}yssQq> zc2%2_q{AZQNpxEs!$*f8%_WPkM>;hO@U02!_r7TdR2L(AYUYLFJ+O8KL zKkv5uN^a3^za}vOR`+=?kLF`q0Kg!iW5mnP$?NyGj5<5u`en~9l6c)$`n?#&3Y4{^ zv8QT}ZTUM)fj(d-pAL!{=r&c*E;K;~!oW$jw6XD&nP2#p3nwP%DmUy_|1%7ImwrtW zkM`*}SaRHc@Y8@D5>5X#ZCNnV#~B1SfTgkGr*VK{(WJr3xeoXs^&XsKa*}N@Z7>u< z3o^&RC4T)T4grR)JVZklVa)PC-|#&vl+lFt5Ze-%Y4dyAA6t+t+vaQYKb_=HH4hJO}cmb6l z`g%;@{2?J#*VvoH%e_uuOe^4xeijdots(41`qLLC+{(Av&1bG(AYqJ0!k-IR`BR)d z5{fe=jlQS{y^a=8@9h;Fdq(#_Zj6tulu8@S+!e(AChFb0J4ZN25+>wrKU8$zh;raJV$Q8cGdQEJ$(zM!rWz?+00*mp#Ql58BN%Xn#UW^_#NO7(Dez^JSBfjS}JA3tr z&d~=wAve;2!8&{U_CvP0xF4ObimB^Eefy|QmL&EZDOsKGs($N8DzbF>-`_|vnz`*H z_>mwlpb-xardsgnu*AvCr|pA(L%t@B=h0)#Q3&CvheUVb-+XQiKe9-$Xe(PeVbWq3 zziwR3;S|eY9SfCgFUYE0YR9G-B4LjKfXVq8FH1TJ;Nr{ZVw=Si=S`sguyhBOk@O=c z$0JzXzPj@Q_;YZup!RZp9Xvzz6tFv5aQ5#s1wnBl2s^$asW*sROpC6#gM(m?J5Kx%+UtHg-T5()YZpFT&9+6b5|^zY>UI$62iPiGD&KE#g^R zW%F0c6%08>`Re$l#&I;dI!NFrhcQkmJzi{iRwR&L`+XC^ZO`N!Y`#fKWwz8hrhc$u zz60#YkUZ9|W#{*8D^2S`3!z#*ixJ_?RjCF7`&U}YtJGsfat`#*ERpsP#TU-tc(|;E z=3Y=PBbW!89e+lxvgQQiUWF#nQ(O=?W*S^B#^Mpdz|@0d{kiBZg^_)_eF$@~0<%+S zhO<9Y;B`2$+2eRm5OG$ng7dJrnji`0^YO3vFH9O}? z$!9d@=cgxW($I{0e+o(!!By%gPwk4l!L8CgJdWaO3vNbOvu;)Cx*KQl`+kmaX)%fo zmm)^^#yLWrIU^>-AK2&rSxW(0O=Uc-47(S|Upb|dXq@~Z;WE_9MKSH$hUJ+8?4@{))rxA*G)a!+vdH{t2tq^4QZN%G2}P$#%n^ z`Fr2Z@DyK)de=?FZrf9Z*nPZUe9yEIz%RTav7;?WCSI~FUMBp7?sV*9e=&R}k%E~g zRPg5u^&FlKM=p-!Lh2|vrqRy8&EHp#WroHMfA__BB@koXdCvuiIL6s%FAcGM$mQst zNbVTvYp#70hsDw-NqGOqE5>v^Od?|Nz|dChm?UNB^yJTNVGq{U)G9J``?^j@C+RsnlM={8O=s!r$w*Sz&3*N};LGbqjz`izu*{F! zyZ-&wD`4FKyf^Y(yiAu-UK2r|&o!3Wr&OVic0pyXrF?)y4J*}WS*)OC1wm=)qgzrUb`ml&!?3~bw|QJlF0wI2t%*7JXOfj z-av>lYCE)7$4-B4tr!rE@w}Cape`GlRL_ROjBAjzQqmHNBJ@fs%J|@mNkkgOyjM^v zU;BnFS|>uwK11XxwQN3|%~=bK^8A#+BR2)(UV~+OF?i~<^Vjc4g?eZ~x~B32-h18* zZ%@TYw6YnP?6mz*HQeBqp)Mh(X?S_LsE+P&kPyI#_7>DHI(PDu87%iey|9BEG^6;% z1+6Kqu3T5{)eo#EtnWHyD?5!wqGA)x5)5sK8Tw*52}cFEUSbMznEuQs=8?Kgaje@P zq*fegB7O0!#fg=LB#yNs{1w7%cGnu z{ypUU!w)LdXoaZD-O?fZvthmQ`;Syf#!d6}G4Ib*@~hWv&v7S22&j?%v8*#_QW+VY z-L+aicBw0vIGUx;Q&i5DX!UvIUq3Q>X(%*Vb5rDh+H6*?VU8*h&LM1KEJDiAAs(2hiL5Qs_iE6 z*`D{+YiGW=#{%o*2<)4=mS<)6oXE^X5FW=mKl{VcY}MKjz=%#YXv=u&uq&_-;IwfB zT`?$NUpDzpylI-od6gt3wv)WF>DeG&hUP<6F$H`ruGP@b`W^&n zUBy0E|KrH_QKosOS&g^?y?jgG9HQhno~rXUvS$qGuS(KM_McA-2ggsH&J0$#J?kH6zA63U z{>pGmt9-s^q~d{aS-%G%2D+K@6H~9Ojvt-u?JX;?{CoFlAryt(@63PLQEFKiyB%~* z$p&Mh*LJB_HQVxR{8He{D;QLuklH8XT=1Asd+>O=O46M!Ph2PCE@O?D;LXl>-5)N_Q^?(C`r+DQXcuD zOBAY;HmER9PvqGxZ@N_^NUVfD#!p0{-+q~Dbp7IgsJ?m!*olPa9&l2d&MU zt*8Ue31|nA&*(}DD7LXMt@L3CXCx&4u>b0iX)em z-lDVqJag&!5<@hWOoUf&doFu*=S?j=34Auc-+N8tO+f%)@F>MF5LoG97kR>0!}&UM z;mDz7k(hMY@P|z2lJ+!TOi)e*L zl@iP*6>QjR?a3Gi;K?IDh8W2`QGcaVxj%s@_04ySsmRLb7Pjck3x+!&x$yGPFwU%qBa(Eq44dsc5>7z(RZ~s&ATD_$ zwYQaX!n1fkE~jMe_|jy`i(*g3PcPX@S8AA?8s=1oBJ!NNAu7ETQ$kYs2IAG%!@!f= z&${tgSyLIjRYM<>eV?4r@0q-NFH-OjHH(f-zwsD2g}=&XA%+?{2|GU#I0D72n5R2* zxIg|N!p7@*%_zxi=#k=JYT7WIpo@nzQSSPT)%9|@uUlRw@}BuhFp+Yq;STegkq(Ik zV{*h^#XN#~fVr2hK>NL4C`yx> zhEGmCpH~AR+?(Hotv58EWfxidkcR!FzsOHy_=T3z!Q%8%^HBOlCjQI83VtHzjKjk( zv;Ac;3x|RfHnWNE#@}*~*swbTH*z71T1?CA<_~3X%q(p-r!AnRQpD8&@H1vPU0Rsy z04FE?dWzcegx=1$CpO#6Irc-WW=G#Vc#7}#cE3P``b>y@dR3Ob zDps*VGSnir9Vzpc83g>hg+PfYXLvm+R zM8%7+OO|LVW(x+RWDSZ@JoR2^$sbamPhog?!NOf=D^|;d3kQs|ThMN?FGb8mPiH6X zyc&uHbGaYxQ23e#&3SVJ+`)SB7Zzo57KEuee-g5oVCD3Z-ooP=kM82)CWvUjNOd1ty?tHD z;$LuIHAQ4D10-h?>A_?(Uph~W;gW;Nllu~#yplBIGyZw2XzT*1b;BY4w*6XWQ6inO z=%*pSG?hgQ^F-xOaXn~Y+Nqi%freHOC#`Z8$X={nT(eucf-d))3ijN0 zUxby|Ka(unEb3FMurlYO*_W4SANWrM6+(=Iq6k>|^!3MJB>V^rN%XK!xHD6sO!-L- zvsqrcOKBy-q_MKnA~uOEw7Y&P~>%DMsHjt@=4O-Coxp&a8*W^dS04P0C;T3L<=|CA67(9eIZ_OP%k@6)T94X~B@ zC&lc2k|*^MXkTh*cvaEbIPfGnaaYMoxJ@Yf1f@Y`Ta%c!5coVT(sr5(#I{c~$e|YZ z>-Bt(4i1Em)yzbASHG>%TdM)6>4rXY)#Au`R8m2V68@81sL~SJ)bwY-JXc3WenUi_ z&bcM&CztR~nWV-X{|#x>&*MK0T<{dKo8yBUh)Lkh+9#pF6OQ?gX2A@QZg^<0kjJW`i;ydnJav()K0nqL2Mo7 z<%VF0M`EU>f|P`@1V7$Klftc03o4b%%Es2vyrQJqhm{*jc|0oY+=|H?-FNPs!Jysk z?{lyXVpR?K}kwOt$$hFNY!>-*?q1mvy4Prv0~by zcQYT5MDW1}ge5775UgJ6YpyhQ{!mD&2@ofW<;RSAY?QjHvvtd9szyBNex|#hH zE)eLIYbeAGUy+%fl_^$>wLQBEZbU$?U}=@JO0^!XkUmrIpj!WUb(}NNz^o-|B5qa{ zH=l16K`kk5M*a?0FwOT6LB45~sI&qFJ?u#%yVGYAi5FZVuvtW0rbxKqA6Dou!)&;0 zTk);2FBXi6ZM=DGYDyRHubswqZnI}ETgs;2$yoPB_=Si_;Duie9E~6dcH4?|FLlQp z5??Q&^w;*V^8Y|U^dQiqGxH3F1Qc|3QsQSln2yX{EGy#;vb2`TlZ*HQvM#5$YbhIcOHi+bopo~l zp&@~5uAbjW984$v%|fd(Rxth9DdEwbst^$9x)V4na*B09+rKRU=z+8dApKns= z!Rj0wYOdS=*wp_We|V|a{?z<_&6y)W;5b4}(W&uj!La$f{I5gaS@5op>k=uq=f+Y} zW4H^Wa`sQ&{t>qR v_Uu1-`x_Vk$l3pEy!|`f@A9hurg;5-1P&kazbrrakM$}1m;B)WcAWnKLA|(g literal 0 HcmV?d00001 diff --git a/pkgs/swarmauri/tests/static/hyperbolic_test_tts.mp3 b/pkgs/swarmauri/tests/static/hyperbolic_test_tts.mp3 new file mode 100644 index 0000000000000000000000000000000000000000..194714d1649c1c74f4cd59df9157095c5495a26c GIT binary patch literal 10005 zcmeI1cTg1Dm&Y3zqU4MM4mk=#&Pj4oh9p^XmLvv1WXMr+6p$!kBq%wEl5-GYP((l^ zXAp@4^uq4`-tNAiuU@^X{p~;dZq?LupSgX$w?A`k-*Zoc==e_H_sz)3&EdjEa`6!Z z01hgEh?JCqf|{CH8ndwzoewBtg^DI zs;0iasi~!KMViI5C31jL3C~>fFVzabrnDWP${4w zIUd0DfIi8X1y7=@!Hc%9j)Y+)M7;suZ%tX&iOPx)n9DtE7WFs1QS7h%pe%ZQ69hl9 zU#UVp3;@B^fxAm9@7|hsXXr)7xS0DsI`kL$4ZFdAH7I6p0C;Mk93Al<1P>qbzf1E= zCW6_o>rGW&==f6#Iw(#|BOPZPh6w5MNfqxyTbQB>&_pk4ShRab8+{y4K+3IVX2|jKiH*GAXUNBTm$dkiYjQD z&W0zOaMJ52M2KX(K|m}8NCygtMqQnbdSZx3dq*>48elViTDPIwh;E{xtum%`!qoz< zd~Q|lbbQt>0z6*khN~FDq<9t$>JO60U}Q11PESjxAL@ z(zw(t7s2laShvOlz-ZaE%*TQ7rY;M0hDo%$P4e4K;q+A7{DMeq!ZDo@k8)7--ZC_p zn-=~=4+88wl~G#TOE$pvd*t)jzjCrZtJ~GWTbLIo5JH#(`Gy+IrSI#y)Y-$j`2>sh zLWeXB$VXi05ONB)$F%ho5yS)61zsxZ2#H{{s;$sILy`uiwE(RvJc}ayc@I!O>ILdW zB$BPHW+oEo9JRDR)HB(|3{82gEMGsi&;VMJ%)WiAk2WdIHuNF8)Y%0TPO&d_fV4oC z!XyB29lXVio(Qx2+zIPv=wUR_)Rt&COZ)MH-}hSh^}~CnG>QxJg5eV?-$(;vRY_%^ zX_9bww!W=x)CpXOH0}wVT#cTt)@xR$=qh;)yQ(H~`04X!5bebbuPSQthC@U^B_q8IKI7*X3QB5Oq^L>c>S3D73!k1VrT)BZ6wbYTr5HIfBezIx^ zbp=ubc&=5qw&QahT|RUqK8}~DiOI>kpJiPl%vL+%R?KALep7Z2r^xczDG z#LD7iPn4q4q~xno5igqI?F;FbhLP@tX0=Wk-0atCX)X8Ix6Fh3#YqSbNEO!@H7#Sz*+Y{{L(=};h>Q)qSR3{b^i(M_o8Ly) zsf3+m!rz=a`xYp)V-<8k2BXI4$V1DU^| z_ptO>uZ2nCwwv!OXOzhU<_x1bpQ36y@l>Pcs0?poTJuNVYn_ndeZeY+Scs$MRlfL* z@lR@W%tk@5#Lgt{dsj?6yrbP~g&5X?ZCDKJpc_$)w8<*Egj%u6X=dNTFuqhD8|n{a z6}q*rrnX3KWCSdJ%oiY(#}Q~#FI3N)QcUkD1~T+`;4~n3?r7Rq z_gH=obCfk9JAjgL*vDiY4kdfEEx zvQiW`-L9vql3a^bA#;ih{g{a(^J!0OFLiHsFO?Q1R%{eA9(Otkxs2>31bj2oV1te{ z?@O6WV|dKwTEI~a9(i;XkA*E~dnp6wk}-I8nziRMHylt!bDKrpJ7Qbz=TPC>>0mwf zo`zoIHdZW>(S-JKCLAAzge0{X^JHeo@WH1LOXH3xMATPS-pX3^pao5{tmMOc>eoIg zkwr)&cl(}sKH|LAFkVOD4UNK9#<>pT;{ND96vy;oanXM(f|Q1m+E{5_k>(^IYkB79 zO+33-%?mDEW@{U>=Y#iK5|qsq8|mBU@E}5)JDoL7l=h3MR0?aMJq_}tYrtGLAz_WA zjwMT8L-oiH^VZwXqqpn_B0j7uahP;&vmxhM-h8KhVydOt(Af~Ah@rr6t8IwGHC0-{c)T|j_s^Jyu38_j|o#b@swYqGUi6DhV zz0`|E`0$}1p=a=57*keR2&&d(%c*X~3a-jAML_SVX5Ph%OAp=fQa}|3*?A8&f zxRm5muv|4);fM`Sf;Ol}M<@e7i~p@1J4o26RHGhM7gFsGrGB>U0oQD^4sr?-g$>Vr1aZL;kZ~SQT*L$s=r6ohqRE8tDmQ={obG+#Mk}&>@vB%?1i5<%f#A@C;@^TN4HNR6u zku$eP@lTh>r~h=d94}70$dX>ztAbq4xd`K);OeC!myd!pPRo37$x8?;0127hy$xvP z>X)*OmJq7_fbeH}`f7yR#HFbse9_RdyC@IcEFVRUJ53z35E*EL$sPf&xns9JwBlyCl5RF9YH%1;R`ReX=jr2e+lQ1VviUB*SZxy<*6xA;=&R%41G*nMqVKQ6M})i%|#?IedTwR z2_e40e+L}1(`F}$d+X_!oQct_LtRp&Yk2T31sO#^mpjVY-X0cshlhmEEhbq&9{8&F zuru9M)}?nij$q~MrUWw6HJOfvYIMwrj-~>}W&19BFGXFumW0tGzJLBE6Ejlo_`A`{ zVTIeAXP~}!x^fP9{_4&{ukDL}D!sHM=k`bb7u>f)*v*c_6cM*%bQ2qEkNMKNO9E$< zX^vS?8OBD|ogR5SXO#wsZ;`!=r?Z2L?zIJLx0;LJmuS|0|3)u!YyIG8yJ^wU)O86& zr-(1!$vpu|lbg)HGyITqo2p6FD0V;X>(07J0iZ$Q4EGMFFCi=gB-#Gv8*i>yb-r?G zNv6J6QR5C)mkiAeBg^D$ka+m%I%b|d#rxbIG=0x?O+`PY$H?nF_z($yC7@!*+sk9~= ze{~aCA%e>5HkXW3AcVpcP^jkTAQ;u;egWhB`1~E)Oe*|)Xa#ZEttAs(qO!4J1}7O>9&1Cj#G1U#i8KkigGa>sDe#51+Fliz#688d7ZV@M1Mn5x)`ZR?$aNy|l@c3#1r(?`oFETQR~5gqD6dlBo6?BdLur0KJm zdEa2A78p_z@Fk|i;{4$2zV=4@2wtqV1OZxRZyqI6?j$3}kj@X|g14LrtPTik+6Gq6vtx_pd4!8mjRe#fq=y4`4TR%%B zkRdNn+koFdy{1UHi4$;u!nX|)3ct&YNg{E0)eF)Q+i;^sa=62Y^mW~&v*NWmJMB03 zaU#mFloa*5HIW7`UYZF^DKeeQeYN0KHD6v+5Ii&XhVrtG2Y}}jhWiw9*Y)@E^M$5% zwd{9kXv{>EohWLjznU&L>cqaf@7#YsB6u&1%6Zhp5*XDY=K!~?K3g{Bmgtf*GrBR+ zcM*H4#1T}CZ%Lp6sZ%0W^~*C}QZL?^ewZmhtdoiiXP0w$D1$-Y8dk+n$Spg3vos_) zKdpEV8kt_p%oVWj{1!bTVi;Sorbk<#(=K#fsJ76ByYUjj5rUj4$7+L(#OaHG>sc_t zjjTdO`RBM@3U)0!G&qt3DW3w)L{R`HLT~q)C6-`xw~R+yKCpw&sq*S5Tvgq=eS%o{ zsETLTQF7@}wJEp@>y0Qr0g?AaezDcAtq4ODFCL<{oRQPKzxW<8UuyaOq$=NrRj&k{ zY-Y*O2s*t$#rc|usq{V>R2qh5ET;aY^6aeY{ZtVQmBr@fgl(N$I$I(S0N56Oi8mpw z`a)%@W_X+5b?{2l^4wgmT14-9(V|=J#-g%Y)F!Ug_SYJD=E(GZE19=on`3RuXP`Z# zf4J23EZ>B&z;my4F=IIZM5AQhy?gpp?69U;MBtU5$t8ph1j%Z+%?2A`-Vx7+e;8G5 zfi|VjEtcGu()aylP7BEAxsGTBmu}qo&JxS>-lQbhI!fbW3-tcL#u5i4Ad`t~xQW#-VrZCer20?S{7tH}^dH&E<1s5kZn+Y*3awHV3|0mq}&H zdZS1b|E)CV>&RIcIax*(dUL!ghcjue{ybpBGEf2H|0!ogtR;=NNow*6XT8fcW}H`^ zxH5Lg3o=+nZoQnqch)N4m1k(bwf&b^4>LSlTT$k_!cMlXqO{Gq6qTXwhhpsJ=BSkl z3YbR>>TMAf$JtL6-Ihc0crWA7TRpdzvkV3_^b}iA4h&8MhHV3t#5UD)7-5ttMw=vxP4NsWuYfydX{|&VJnD)=&s&Idd2E=u5(K&e#=X9 z1&1p;7Cvhjm;iQqj$2HUQ-nC1V$=%}p6Iw(r<>N~EA8~SM zoW?i0gRrV~wXz=&b_*+ch1)Ut@~IC2t_6qOWU zM;RL#IQJmkSZOi|G$CagU41%{Z%=nV;oYYfC2%%0V7R_ZX2bP9qc$~}7(V}eDkG)2 zXc60b>qfb~n`ramJ@ynsIS_0JSO3}Y(MEgX+<^SwAs|RXxwJQFJ)IU_ubw62pOrX& z8FhaewtMjaz1jS8+%Ysll*w23ggsbYu-c@6%nF%M1pP@&lA~nJR&n4I<+x<-SW+%-iPQRltP(&jR$FeL>!&WHq+h>&o%6ruj96BI$WN2HEfd;=Vy2!9dp=%xF!_HhX{ z!Bgd)h$zlnQ@g!e3Ul^Z;RY}(QgVm5dsRw;be9m;5kwnk_y()?cR7LWGfdOZGoHfV} z*0m>;_sWM|)zrAjiqG!$q8y^68+z8KwwIU!3(p(0G(v2IuN`44|5U0heD6=-p3f$O z;*Em-lc8ucUj=tfrJ%MCb~r!9Km|;)$m;f^d15IGcVqIj!XFv0P_{?Ebb~%q(S=xO3E>bV7L3V9j|CAf=~}hHlT~9?i`KB{Shcb z3ymhap_k4+qO$~;XXj(TCKz|?$$xt7;CWRGr~|N0Rm`cxU_fOS9cRj8H%J~MY0`&Xq}5KJ7EHg!UnCUl~1Ch^v?5`H$}Nt^@13x zdM$PJ2Bo!64jXiKsKBddtp4YK~_0uge4w+Qf5HdRqaM8-s!kCs7hVigI%CrGZnEeCZRrs3*-ZGgW+}=cdEC*{0(o9h0=;g)5T*Kd$oU5v|F(NfC!@a z`q6rU`^VO9Td1t9PXK)5;?W8cJUPPXq+-fM83&$!}M~{;Rlt zd31R{y9kY%Vv#cGek+66BAGCvl>gb~znz6ZFG{h1DZPMN+bjrnGy$}DbiEwPAb9M| zXmzVJiT@0F0Fs;){yE&Hx_6g+Fi=kttsGo6BER@_%mv(Xh^A z1iqRwf0=Y|DIpmv>uzAf?>iv?4ACI|%kd)l!GhUOcW3WR>-aeT6`kL^{Eu@e)-)On zIOJE3J_GN})|VLk=JN8UFDuOlfJ)~Bj^aXBs2@f_;a&{zM>PJL{hxGp0maAow4E^k zV2@!_Xao6I=`05-Y+SkS5CMg1T~yBZfPax@}vOdulf6<-v6XCfz=9N zoL}Uw7c*DxVut;#^GkGpbNeNhKQ;c%?N1T?vcuopeu?Ezjem3dQ$)Y)@He+#V);|! k{}s3YF_6nV@n5uV|Nqb_{EvJ`81cWB@BEtEzgXwr0PHrJQ2+n{ literal 0 HcmV?d00001 diff --git a/pkgs/swarmauri/tests/unit/llms/HyperbolicAudioTTS_unit_test.py b/pkgs/swarmauri/tests/unit/llms/HyperbolicAudioTTS_unit_test.py new file mode 100644 index 000000000..9b81c84ef --- /dev/null +++ b/pkgs/swarmauri/tests/unit/llms/HyperbolicAudioTTS_unit_test.py @@ -0,0 +1,141 @@ +import logging +import pytest +import os + +from swarmauri.llms.concrete.HyperbolicAudioTTS import HyperbolicAudioTTS as LLM +from dotenv import load_dotenv +from swarmauri.utils.timeout_wrapper import timeout +from pathlib import Path + +load_dotenv() + +API_KEY = os.getenv("HYPERBOLIC_API_KEY") + + +# Get the current working directory +root_dir = Path(__file__).resolve().parents[2] + +# Construct file paths dynamically +file_path = os.path.join(root_dir, "static", "hyperbolic_test_tts.mp3") +file_path2 = os.path.join(root_dir, "static", "hyperbolic_test2.mp3") +file_path3 = os.path.join(root_dir, "static", "hyperbolic_test3.mp3") + + +@pytest.fixture(scope="module") +def hyperbolic_model(): + if not API_KEY: + pytest.skip("Skipping due to environment variable not set") + llm = LLM(api_key=API_KEY) + return llm + + +@timeout(5) +def get_allowed_languages(): + if not API_KEY: + return [] + llm = LLM(api_key=API_KEY) + return llm.allowed_languages + + +@timeout(5) +@pytest.mark.unit +def test_ubc_resource(hyperbolic_model): + assert hyperbolic_model.resource == "LLM" + + +@timeout(5) +@pytest.mark.unit +def test_ubc_type(hyperbolic_model): + assert hyperbolic_model.type == "HyperbolicAudioTTS" + + +@timeout(5) +@pytest.mark.unit +def test_serialization(hyperbolic_model): + assert ( + hyperbolic_model.id + == LLM.model_validate_json(hyperbolic_model.model_dump_json()).id + ) + + +@timeout(5) +@pytest.mark.unit +def test_default_speed(hyperbolic_model): + assert hyperbolic_model.speed == 1.0 + + +@timeout(5) +@pytest.mark.parametrize("language", get_allowed_languages()) +@pytest.mark.unit +def test_predict(hyperbolic_model, language): + """ + Test prediction with different languages + Note: Adjust the text according to the language if needed + """ + # Set the language for the test + hyperbolic_model.language = language + + # Select an appropriate text based on the language + texts = { + "EN": "Hello, this is a test of text-to-speech output in English.", + "ES": "Hola, esta es una prueba de salida de texto a voz en español.", + "FR": "Bonjour, ceci est un test de sortie de texte en français.", + "ZH": "这是一个中文语音转换测试。", + "JP": "これは日本語の音声合成テストです。", + "KR": "이것은 한국어 음성 합성 테스트입니다.", + } + + text = texts.get( + language, "Hello, this is a generic test of text-to-speech output." + ) + + audio_file_path = hyperbolic_model.predict(text=text, audio_path=file_path) + + logging.info(audio_file_path) + + assert isinstance(audio_file_path, str) + assert os.path.exists(audio_file_path) + assert os.path.getsize(audio_file_path) > 0 + + +@timeout(5) +@pytest.mark.unit +def test_batch(hyperbolic_model): + """ + Test batch processing of multiple texts + """ + text_path_dict = { + "Hello": file_path, + "Hi there": file_path2, + "Good morning": file_path3, + } + + results = hyperbolic_model.batch(text_path_dict=text_path_dict) + assert len(results) == len(text_path_dict) + + for result in results: + assert isinstance(result, str) + assert os.path.exists(result) + assert os.path.getsize(result) > 0 + + +@timeout(5) +@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.unit +async def test_abatch(hyperbolic_model): + """ + Test asynchronous batch processing of multiple texts + """ + text_path_dict = { + "Hello": file_path, + "Hi there": file_path2, + "Good morning": file_path3, + } + + results = await hyperbolic_model.abatch(text_path_dict=text_path_dict) + assert len(results) == len(text_path_dict) + + for result in results: + assert isinstance(result, str) + assert os.path.exists(result) + assert os.path.getsize(result) > 0 From e2005640bfb576901eb802d15511fca97e74d354 Mon Sep 17 00:00:00 2001 From: michaeldecent2 <111002205+MichaelDecent@users.noreply.github.com> Date: Wed, 20 Nov 2024 12:46:21 +0100 Subject: [PATCH 006/168] comm - changed test files path --- .../embeddings/Doc2VecEmbedding_unit_test.py | 22 ++++ .../parsers/TextBlobNounParser_unit_test.py | 0 .../TextBlobSentenceParser_unit_test.py | 0 .../tests/unit/tools/TextLength_unit_test.py | 0 .../Doc2VecVectorStore_unit_test.py | 72 +++++++++++ .../swarmauri/parsers/concrete/__init__.py | 2 +- .../swarmauri/tools/base/ToolBase.py | 3 + .../swarmauri/tools/concrete/SMOGIndexTool.py | 113 ------------------ .../utils/extract_signature_decorator.py | 85 +++++++++++++ .../vector_stores/concrete/__init__.py | 2 +- 10 files changed, 184 insertions(+), 115 deletions(-) create mode 100644 pkgs/community/tests/unit/embeddings/Doc2VecEmbedding_unit_test.py rename pkgs/{swarmauri => community}/tests/unit/parsers/TextBlobNounParser_unit_test.py (100%) rename pkgs/{swarmauri => community}/tests/unit/parsers/TextBlobSentenceParser_unit_test.py (100%) rename pkgs/{swarmauri => community}/tests/unit/tools/TextLength_unit_test.py (100%) create mode 100644 pkgs/community/tests/unit/vector_stores/Doc2VecVectorStore_unit_test.py delete mode 100644 pkgs/swarmauri/swarmauri/tools/concrete/SMOGIndexTool.py create mode 100644 pkgs/swarmauri/swarmauri/utils/extract_signature_decorator.py diff --git a/pkgs/community/tests/unit/embeddings/Doc2VecEmbedding_unit_test.py b/pkgs/community/tests/unit/embeddings/Doc2VecEmbedding_unit_test.py new file mode 100644 index 000000000..7f3afc447 --- /dev/null +++ b/pkgs/community/tests/unit/embeddings/Doc2VecEmbedding_unit_test.py @@ -0,0 +1,22 @@ +import pytest +from swarmauri_community.embeddings.concrete.Doc2VecEmbedding import Doc2VecEmbedding + +@pytest.mark.unit +def test_ubc_resource(): + assert Doc2VecEmbedding().resource == 'Embedding' + +@pytest.mark.unit +def test_ubc_type(): + assert Doc2VecEmbedding().type == 'Doc2VecEmbedding' + +@pytest.mark.unit +def test_serialization(): + embedder = Doc2VecEmbedding() + assert embedder.id == Doc2VecEmbedding.model_validate_json(embedder.model_dump_json()).id + +@pytest.mark.unit +def test_fit_transform(): + embedder = Doc2VecEmbedding() + documents = ['test', 'cat', 'banana'] + embedder.fit_transform(documents) + assert ['banana', 'cat', 'test'] == embedder.extract_features() \ No newline at end of file diff --git a/pkgs/swarmauri/tests/unit/parsers/TextBlobNounParser_unit_test.py b/pkgs/community/tests/unit/parsers/TextBlobNounParser_unit_test.py similarity index 100% rename from pkgs/swarmauri/tests/unit/parsers/TextBlobNounParser_unit_test.py rename to pkgs/community/tests/unit/parsers/TextBlobNounParser_unit_test.py diff --git a/pkgs/swarmauri/tests/unit/parsers/TextBlobSentenceParser_unit_test.py b/pkgs/community/tests/unit/parsers/TextBlobSentenceParser_unit_test.py similarity index 100% rename from pkgs/swarmauri/tests/unit/parsers/TextBlobSentenceParser_unit_test.py rename to pkgs/community/tests/unit/parsers/TextBlobSentenceParser_unit_test.py diff --git a/pkgs/swarmauri/tests/unit/tools/TextLength_unit_test.py b/pkgs/community/tests/unit/tools/TextLength_unit_test.py similarity index 100% rename from pkgs/swarmauri/tests/unit/tools/TextLength_unit_test.py rename to pkgs/community/tests/unit/tools/TextLength_unit_test.py diff --git a/pkgs/community/tests/unit/vector_stores/Doc2VecVectorStore_unit_test.py b/pkgs/community/tests/unit/vector_stores/Doc2VecVectorStore_unit_test.py new file mode 100644 index 000000000..01c8eb634 --- /dev/null +++ b/pkgs/community/tests/unit/vector_stores/Doc2VecVectorStore_unit_test.py @@ -0,0 +1,72 @@ +import pytest +from swarmauri.documents.concrete.Document import Document +from swarmauri_community.vector_stores.Doc2VecVectorStore import Doc2VecVectorStore + + +@pytest.mark.unit +def test_ubc_resource(): + vs = Doc2VecVectorStore() + assert vs.resource == "VectorStore" + assert vs.embedder.resource == "Embedding" + + +@pytest.mark.unit +def test_ubc_type(): + vs = Doc2VecVectorStore() + assert vs.type == "Doc2VecVectorStore" + + +@pytest.mark.unit +def test_serialization(): + vs = Doc2VecVectorStore() + assert vs.id == Doc2VecVectorStore.model_validate_json(vs.model_dump_json()).id + + +@pytest.mark.unit +def test_top_k(): + vs = Doc2VecVectorStore() + documents = [ + Document(content="test"), + Document(content="test1"), + Document(content="test2"), + Document(content="test3"), + ] + + vs.add_documents(documents) + assert len(vs.retrieve(query="test", top_k=2)) == 2 + + +@pytest.mark.unit +def test_adding_more_doc(): + vs = Doc2VecVectorStore() + documents_batch_1 = [ + Document(content="test"), + Document(content="test1"), + Document(content="test2"), + Document(content="test3"), + ] + documents_batch_2 = [ + Document(content="This is a test. Test number 4"), + Document(content="This is a test. Test number 5"), + Document(content="This is a test. Test number 6"), + Document(content="This is a test. Test number 7"), + ] + doc_count = len(documents_batch_1) + len(documents_batch_2) + + vs.add_documents(documents_batch_1) + vs.add_documents(documents_batch_2) + assert len(vs.retrieve(query="test", top_k=doc_count)) == doc_count + + +@pytest.mark.unit +def test_oov(): + """Test for Out Of Vocabulary (OOV) words""" + vs = Doc2VecVectorStore() + documents = [ + Document(content="test"), + Document(content="test1"), + Document(content="test2"), + Document(content="test3"), + ] + vs.add_documents(documents) + assert len(vs.retrieve(query="what is test 4", top_k=2)) == 2 diff --git a/pkgs/swarmauri/swarmauri/parsers/concrete/__init__.py b/pkgs/swarmauri/swarmauri/parsers/concrete/__init__.py index 45b1c7640..a150d0de5 100644 --- a/pkgs/swarmauri/swarmauri/parsers/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/parsers/concrete/__init__.py @@ -24,7 +24,7 @@ def _lazy_import(module_name, module_description=None): "PythonParser", "RegExParser", "TextBlobNounParser", - "TextBlobSentenceParser", + # "TextBlobSentenceParser", "URLExtractorParser", "XMLParser", ] diff --git a/pkgs/swarmauri/swarmauri/tools/base/ToolBase.py b/pkgs/swarmauri/swarmauri/tools/base/ToolBase.py index 3dfade766..dedd82244 100644 --- a/pkgs/swarmauri/swarmauri/tools/base/ToolBase.py +++ b/pkgs/swarmauri/swarmauri/tools/base/ToolBase.py @@ -19,6 +19,9 @@ def call(self, *args, **kwargs): @abstractmethod def __call__(self, *args, **kwargs): raise NotImplementedError("Subclasses must implement the __call__ method.") + + def extract_signature_details(self): + pass # #def __getstate__(self): diff --git a/pkgs/swarmauri/swarmauri/tools/concrete/SMOGIndexTool.py b/pkgs/swarmauri/swarmauri/tools/concrete/SMOGIndexTool.py deleted file mode 100644 index 23ce384df..000000000 --- a/pkgs/swarmauri/swarmauri/tools/concrete/SMOGIndexTool.py +++ /dev/null @@ -1,113 +0,0 @@ -from swarmauri_core.typing import SubclassUnion -from typing import List, Literal, Dict -from pydantic import Field -from swarmauri.tools.base.ToolBase import ToolBase -from swarmauri.tools.concrete.Parameter import Parameter -import re -import math -import nltk -from nltk.tokenize import sent_tokenize - -# Download required NLTK data once during module load -nltk.download("punkt", quiet=True) - - -class SMOGIndexTool(ToolBase): - version: str = "0.1.dev2" - parameters: List[Parameter] = Field( - default_factory=lambda: [ - Parameter( - name="text", - type="string", - description="The text to analyze for SMOG Index", - required=True, - ) - ] - ) - name: str = "SMOGIndexTool" - description: str = "Calculates the SMOG Index for the provided text." - type: Literal["SMOGIndexTool"] = "SMOGIndexTool" - - def __call__(self, text: str) -> Dict[str, float]: - """ - Calculates the SMOG Index for the provided text. - - Parameters: - text (str): The text to analyze. - - Returns: - float: The calculated SMOG Index. - """ - return {"smog_index": self.calculate_smog_index(text)} - - def calculate_smog_index(self, text: str) -> float: - """ - Calculate the SMOG Index for a given text. - - Parameters: - text (str): The text to analyze. - - Returns: - float: The calculated SMOG Index. - """ - sentences = self.count_sentences(text) - polysyllables = self.count_polysyllables(text) - - if sentences == 0: - return 0.0 # Avoid division by zero - - smog_index = 1.0430 * math.sqrt(polysyllables * (30 / sentences)) + 3.1291 - return round(smog_index, 1) - - def count_sentences(self, text: str) -> int: - """ - Count the number of sentences in the text. - - Parameters: - text (str): The text to analyze. - - Returns: - int: The number of sentences in the text. - """ - sentences = sent_tokenize(text) - return len(sentences) - - def count_polysyllables(self, text: str) -> int: - """ - Count the number of polysyllabic words (words with three or more syllables) in the text. - - Parameters: - text (str): The text to analyze. - - Returns: - int: The number of polysyllabic words in the text. - """ - words = re.findall(r"\w+", text) - return len([word for word in words if self.count_syllables(word) >= 3]) - - def count_syllables(self, word: str) -> int: - """ - Count the number of syllables in a given word. - - Parameters: - word (str): The word to analyze. - - Returns: - int: The number of syllables in the word. - """ - word = word.lower() - vowels = "aeiouy" - count = 0 - if word and word[0] in vowels: - count += 1 - for index in range(1, len(word)): - if word[index] in vowels and word[index - 1] not in vowels: - count += 1 - if word.endswith("e") and not word.endswith("le"): - count -= 1 - if count == 0: - count = 1 - return count - - -SubclassUnion.update(baseclass=ToolBase, type_name="SMOGIndexTool", obj=SMOGIndexTool) diff --git a/pkgs/swarmauri/swarmauri/utils/extract_signature_decorator.py b/pkgs/swarmauri/swarmauri/utils/extract_signature_decorator.py new file mode 100644 index 000000000..412d10b90 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/utils/extract_signature_decorator.py @@ -0,0 +1,85 @@ +from typing import ( + List, + Any, + Union, + Optional, + Callable, + get_type_hints, + get_args, + get_origin, +) +import inspect +from pydantic import BaseModel +from swarmauri.tools.concrete.Parameter import Parameter + + +class MethodSignatureExtractor(BaseModel): + parameters: List[Parameter] = [] + method: Callable + _type_mapping: dict = { + int: "integer", + float: "number", + str: "string", + bool: "boolean", + list: "array", + dict: "object", + Any: "any", + } + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.parameters = self.extract_signature_details() + + def _python_type_to_json_schema_type(self, py_type): + if get_origin(py_type) is not None: + origin = get_origin(py_type) + args = get_args(py_type) + + if origin is list: + items_type = self._python_type_to_json_schema_type(args[0]) + return {"type": "array", "items": items_type} + elif origin is dict: + return {"type": "object"} + elif origin in (Union, Optional): + if len(args) == 2 and type(None) in args: + non_none_type = args[0] if args[1] is type(None) else args[1] + return self._python_type_to_json_schema_type(non_none_type) + return { + "oneOf": [ + self._python_type_to_json_schema_type(arg) for arg in args + ] + } + return {"type": self._type_mapping.get(origin, "string")} + else: + return {"type": self._type_mapping.get(py_type, "string")} + + def extract_signature_details(self): + sig = inspect.signature(self.method) + type_hints = get_type_hints(self.method) + parameters = sig.parameters + details_list = [] + + for param_name, param in parameters.items(): + if param_name == "self": + continue + + param_type = type_hints.get(param_name, Any) + param_default = ( + param.default if param.default is not inspect.Parameter.empty else None + ) + required = param.default is inspect.Parameter.empty + enum = None + param_type_json_schema = self._python_type_to_json_schema_type(param_type) + + description = f"Parameter {param_name} of type {param_type_json_schema}" + + detail = Parameter( + name=param_name, + type=param_type_json_schema["type"], + description=description, + required=required, + enum=enum, + ) + details_list.append(detail) + + return details_list diff --git a/pkgs/swarmauri/swarmauri/vector_stores/concrete/__init__.py b/pkgs/swarmauri/swarmauri/vector_stores/concrete/__init__.py index 08a36e26c..7aee51192 100644 --- a/pkgs/swarmauri/swarmauri/vector_stores/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/vector_stores/concrete/__init__.py @@ -12,7 +12,7 @@ def _lazy_import(module_name, module_description=None): # List of vector store names (file names without the ".py" extension) vector_store_files = [ - "Doc2VecVectorStore", + # "Doc2VecVectorStore", "MlmVectorStore", "SqliteVectorStore", "TfidfVectorStore", From 22cfec302f4e7441c0e87ba5bb319b3b7eef8f9e Mon Sep 17 00:00:00 2001 From: michaeldecent2 <111002205+MichaelDecent@users.noreply.github.com> Date: Wed, 20 Nov 2024 13:32:13 +0100 Subject: [PATCH 007/168] comm - dependencies fix --- .../embeddings/concrete/MlmEmbedding.py | 0 .../parsers/concrete/BERTEmbeddingParser.py | 0 .../vector_stores}/MlmVectorStore.py | 2 +- .../unit/embeddings/MlmEmbedding_unit_test.py | 2 +- .../vector_stores/MlmVectorStore_unit_test.py | 2 +- pkgs/swarmauri/pyproject.toml | 20 +++--- .../swarmauri/embeddings/concrete/__init__.py | 8 +-- .../swarmauri/parsers/concrete/__init__.py | 4 +- .../vector_stores/concrete/__init__.py | 2 +- .../embeddings/Doc2VecEmbedding_unit_test.py | 22 ------ .../Doc2VecVectorStore_unit_test.py | 72 ------------------- 11 files changed, 20 insertions(+), 114 deletions(-) rename pkgs/{swarmauri/swarmauri => community/swarmauri_community}/embeddings/concrete/MlmEmbedding.py (100%) rename pkgs/{swarmauri/swarmauri => community/swarmauri_community}/parsers/concrete/BERTEmbeddingParser.py (100%) rename pkgs/{swarmauri/swarmauri/vector_stores/concrete => community/swarmauri_community/vector_stores}/MlmVectorStore.py (96%) rename pkgs/{swarmauri => community}/tests/unit/embeddings/MlmEmbedding_unit_test.py (85%) rename pkgs/{swarmauri => community}/tests/unit/vector_stores/MlmVectorStore_unit_test.py (89%) delete mode 100644 pkgs/swarmauri/tests/unit/embeddings/Doc2VecEmbedding_unit_test.py delete mode 100644 pkgs/swarmauri/tests/unit/vector_stores/Doc2VecVectorStore_unit_test.py diff --git a/pkgs/swarmauri/swarmauri/embeddings/concrete/MlmEmbedding.py b/pkgs/community/swarmauri_community/embeddings/concrete/MlmEmbedding.py similarity index 100% rename from pkgs/swarmauri/swarmauri/embeddings/concrete/MlmEmbedding.py rename to pkgs/community/swarmauri_community/embeddings/concrete/MlmEmbedding.py diff --git a/pkgs/swarmauri/swarmauri/parsers/concrete/BERTEmbeddingParser.py b/pkgs/community/swarmauri_community/parsers/concrete/BERTEmbeddingParser.py similarity index 100% rename from pkgs/swarmauri/swarmauri/parsers/concrete/BERTEmbeddingParser.py rename to pkgs/community/swarmauri_community/parsers/concrete/BERTEmbeddingParser.py diff --git a/pkgs/swarmauri/swarmauri/vector_stores/concrete/MlmVectorStore.py b/pkgs/community/swarmauri_community/vector_stores/MlmVectorStore.py similarity index 96% rename from pkgs/swarmauri/swarmauri/vector_stores/concrete/MlmVectorStore.py rename to pkgs/community/swarmauri_community/vector_stores/MlmVectorStore.py index ea5902602..e3e19abf9 100644 --- a/pkgs/swarmauri/swarmauri/vector_stores/concrete/MlmVectorStore.py +++ b/pkgs/community/swarmauri_community/vector_stores/MlmVectorStore.py @@ -1,6 +1,6 @@ from typing import List, Union, Literal from swarmauri.documents.concrete.Document import Document -from swarmauri.embeddings.concrete.MlmEmbedding import MlmEmbedding +from pkgs.community.swarmauri_community.embeddings.concrete.MlmEmbedding import MlmEmbedding from swarmauri.distances.concrete.CosineDistance import CosineDistance from swarmauri.vector_stores.base.VectorStoreBase import VectorStoreBase diff --git a/pkgs/swarmauri/tests/unit/embeddings/MlmEmbedding_unit_test.py b/pkgs/community/tests/unit/embeddings/MlmEmbedding_unit_test.py similarity index 85% rename from pkgs/swarmauri/tests/unit/embeddings/MlmEmbedding_unit_test.py rename to pkgs/community/tests/unit/embeddings/MlmEmbedding_unit_test.py index 6962bb802..399318ebd 100644 --- a/pkgs/swarmauri/tests/unit/embeddings/MlmEmbedding_unit_test.py +++ b/pkgs/community/tests/unit/embeddings/MlmEmbedding_unit_test.py @@ -1,5 +1,5 @@ import pytest -from swarmauri.embeddings.concrete.MlmEmbedding import MlmEmbedding +from pkgs.community.swarmauri_community.embeddings.concrete.MlmEmbedding import MlmEmbedding @pytest.mark.unit def test_ubc_resource(): diff --git a/pkgs/swarmauri/tests/unit/vector_stores/MlmVectorStore_unit_test.py b/pkgs/community/tests/unit/vector_stores/MlmVectorStore_unit_test.py similarity index 89% rename from pkgs/swarmauri/tests/unit/vector_stores/MlmVectorStore_unit_test.py rename to pkgs/community/tests/unit/vector_stores/MlmVectorStore_unit_test.py index 06b0fa263..27e300fe7 100644 --- a/pkgs/swarmauri/tests/unit/vector_stores/MlmVectorStore_unit_test.py +++ b/pkgs/community/tests/unit/vector_stores/MlmVectorStore_unit_test.py @@ -1,6 +1,6 @@ import pytest from swarmauri.documents.concrete.Document import Document -from swarmauri.vector_stores.concrete.MlmVectorStore import MlmVectorStore +from pkgs.community.swarmauri_community.vector_stores.MlmVectorStore import MlmVectorStore @pytest.mark.unit diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index fb20c59d9..04eb35465 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -42,10 +42,10 @@ beautifulsoup4 = { version = "04.12.3", optional = true } scipy = { version = ">=1.7.0,<1.14.0", optional = true } #scikit-learn = { version = "^1.4.2", optional = true } #spacy = { version = ">=3.0.0,<=3.8.2", optional = true } -transformers = { version = "^4.45.0", optional = true } -torch = { version = "^2.5.0", optional = true } -keras = { version = ">=3.2.0", optional = true } -tf-keras = { version = ">=2.16.0", optional = true } +#transformers = { version = "^4.45.0", optional = true } +#torch = { version = "^2.5.0", optional = true } +#keras = { version = ">=3.2.0", optional = true } +#tf-keras = { version = ">=2.16.0", optional = true } matplotlib = { version = ">=3.9.2", optional = true } [tool.poetry.extras] @@ -56,9 +56,9 @@ nlp = ["nltk", "textblob", "yake"] nlp_tools = ["beautifulsoup4"] #ml_toolkits = ["gensim", "scipy", "scikit-learn"] #spacy = ["spacy"] -transformers = ["transformers"] -torch = ["torch"] -tensorflow = ["keras", "tf-keras"] +#transformers = ["transformers"] +#torch = ["torch"] +#tensorflow = ["keras", "tf-keras"] visualization = ["matplotlib"] # Full option to install all extras @@ -70,9 +70,9 @@ full = [ "beautifulsoup4", #"gensim", "scipy", "scikit-learn", #"spacy", - "transformers", - "torch", - "keras", "tf-keras", + #"transformers", + #"torch", + #"keras", "tf-keras", "matplotlib" ] diff --git a/pkgs/swarmauri/swarmauri/embeddings/concrete/__init__.py b/pkgs/swarmauri/swarmauri/embeddings/concrete/__init__.py index a1f0f231c..a4fd73974 100644 --- a/pkgs/swarmauri/swarmauri/embeddings/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/embeddings/concrete/__init__.py @@ -11,20 +11,20 @@ def _lazy_import(module_name, module_description=None): return None # Lazy loading of embeddings with descriptive names -Doc2VecEmbedding = _lazy_import("swarmauri.embeddings.concrete.Doc2VecEmbedding", "Doc2VecEmbedding") +# Doc2VecEmbedding = _lazy_import("swarmauri.embeddings.concrete.Doc2VecEmbedding", "Doc2VecEmbedding") GeminiEmbedding = _lazy_import("swarmauri.embeddings.concrete.GeminiEmbedding", "GeminiEmbedding") MistralEmbedding = _lazy_import("swarmauri.embeddings.concrete.MistralEmbedding", "MistralEmbedding") -MlmEmbedding = _lazy_import("swarmauri.embeddings.concrete.MlmEmbedding", "MlmEmbedding") +# MlmEmbedding = _lazy_import("swarmauri.embeddings.concrete.MlmEmbedding", "MlmEmbedding") NmfEmbedding = _lazy_import("swarmauri.embeddings.concrete.NmfEmbedding", "NmfEmbedding") OpenAIEmbedding = _lazy_import("swarmauri.embeddings.concrete.OpenAIEmbedding", "OpenAIEmbedding") TfidfEmbedding = _lazy_import("swarmauri.embeddings.concrete.TfidfEmbedding", "TfidfEmbedding") # Adding lazy-loaded modules to __all__ __all__ = [ - "Doc2VecEmbedding", + # "Doc2VecEmbedding", "GeminiEmbedding", "MistralEmbedding", - "MlmEmbedding", + # "MlmEmbedding", "NmfEmbedding", "OpenAIEmbedding", "TfidfEmbedding", diff --git a/pkgs/swarmauri/swarmauri/parsers/concrete/__init__.py b/pkgs/swarmauri/swarmauri/parsers/concrete/__init__.py index a150d0de5..a42b1b55f 100644 --- a/pkgs/swarmauri/swarmauri/parsers/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/parsers/concrete/__init__.py @@ -13,7 +13,7 @@ def _lazy_import(module_name, module_description=None): # List of parser names (file names without the ".py" extension) parser_files = [ "BeautifulSoupElementParser", - "BERTEmbeddingParser", + # "BERTEmbeddingParser", "CSVParser", "EntityRecognitionParser", "HTMLTagStripParser", @@ -23,7 +23,7 @@ def _lazy_import(module_name, module_description=None): "PhoneNumberExtractorParser", "PythonParser", "RegExParser", - "TextBlobNounParser", + # "TextBlobNounParser", # "TextBlobSentenceParser", "URLExtractorParser", "XMLParser", diff --git a/pkgs/swarmauri/swarmauri/vector_stores/concrete/__init__.py b/pkgs/swarmauri/swarmauri/vector_stores/concrete/__init__.py index 7aee51192..2f946b377 100644 --- a/pkgs/swarmauri/swarmauri/vector_stores/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/vector_stores/concrete/__init__.py @@ -13,7 +13,7 @@ def _lazy_import(module_name, module_description=None): # List of vector store names (file names without the ".py" extension) vector_store_files = [ # "Doc2VecVectorStore", - "MlmVectorStore", + # "MlmVectorStore", "SqliteVectorStore", "TfidfVectorStore", ] diff --git a/pkgs/swarmauri/tests/unit/embeddings/Doc2VecEmbedding_unit_test.py b/pkgs/swarmauri/tests/unit/embeddings/Doc2VecEmbedding_unit_test.py deleted file mode 100644 index 7f3afc447..000000000 --- a/pkgs/swarmauri/tests/unit/embeddings/Doc2VecEmbedding_unit_test.py +++ /dev/null @@ -1,22 +0,0 @@ -import pytest -from swarmauri_community.embeddings.concrete.Doc2VecEmbedding import Doc2VecEmbedding - -@pytest.mark.unit -def test_ubc_resource(): - assert Doc2VecEmbedding().resource == 'Embedding' - -@pytest.mark.unit -def test_ubc_type(): - assert Doc2VecEmbedding().type == 'Doc2VecEmbedding' - -@pytest.mark.unit -def test_serialization(): - embedder = Doc2VecEmbedding() - assert embedder.id == Doc2VecEmbedding.model_validate_json(embedder.model_dump_json()).id - -@pytest.mark.unit -def test_fit_transform(): - embedder = Doc2VecEmbedding() - documents = ['test', 'cat', 'banana'] - embedder.fit_transform(documents) - assert ['banana', 'cat', 'test'] == embedder.extract_features() \ No newline at end of file diff --git a/pkgs/swarmauri/tests/unit/vector_stores/Doc2VecVectorStore_unit_test.py b/pkgs/swarmauri/tests/unit/vector_stores/Doc2VecVectorStore_unit_test.py deleted file mode 100644 index 01c8eb634..000000000 --- a/pkgs/swarmauri/tests/unit/vector_stores/Doc2VecVectorStore_unit_test.py +++ /dev/null @@ -1,72 +0,0 @@ -import pytest -from swarmauri.documents.concrete.Document import Document -from swarmauri_community.vector_stores.Doc2VecVectorStore import Doc2VecVectorStore - - -@pytest.mark.unit -def test_ubc_resource(): - vs = Doc2VecVectorStore() - assert vs.resource == "VectorStore" - assert vs.embedder.resource == "Embedding" - - -@pytest.mark.unit -def test_ubc_type(): - vs = Doc2VecVectorStore() - assert vs.type == "Doc2VecVectorStore" - - -@pytest.mark.unit -def test_serialization(): - vs = Doc2VecVectorStore() - assert vs.id == Doc2VecVectorStore.model_validate_json(vs.model_dump_json()).id - - -@pytest.mark.unit -def test_top_k(): - vs = Doc2VecVectorStore() - documents = [ - Document(content="test"), - Document(content="test1"), - Document(content="test2"), - Document(content="test3"), - ] - - vs.add_documents(documents) - assert len(vs.retrieve(query="test", top_k=2)) == 2 - - -@pytest.mark.unit -def test_adding_more_doc(): - vs = Doc2VecVectorStore() - documents_batch_1 = [ - Document(content="test"), - Document(content="test1"), - Document(content="test2"), - Document(content="test3"), - ] - documents_batch_2 = [ - Document(content="This is a test. Test number 4"), - Document(content="This is a test. Test number 5"), - Document(content="This is a test. Test number 6"), - Document(content="This is a test. Test number 7"), - ] - doc_count = len(documents_batch_1) + len(documents_batch_2) - - vs.add_documents(documents_batch_1) - vs.add_documents(documents_batch_2) - assert len(vs.retrieve(query="test", top_k=doc_count)) == doc_count - - -@pytest.mark.unit -def test_oov(): - """Test for Out Of Vocabulary (OOV) words""" - vs = Doc2VecVectorStore() - documents = [ - Document(content="test"), - Document(content="test1"), - Document(content="test2"), - Document(content="test3"), - ] - vs.add_documents(documents) - assert len(vs.retrieve(query="what is test 4", top_k=2)) == 2 From f1cd5e210b1e653947fb366f49881bdb54003a8e Mon Sep 17 00:00:00 2001 From: michaeldecent2 <111002205+MichaelDecent@users.noreply.github.com> Date: Wed, 20 Nov 2024 13:39:29 +0100 Subject: [PATCH 008/168] comm - minor fix --- .../swarmauri_community/vector_stores/MlmVectorStore.py | 2 +- pkgs/community/tests/unit/embeddings/MlmEmbedding_unit_test.py | 2 +- .../tests/unit/vector_stores/MlmVectorStore_unit_test.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/community/swarmauri_community/vector_stores/MlmVectorStore.py b/pkgs/community/swarmauri_community/vector_stores/MlmVectorStore.py index e3e19abf9..21f45ec35 100644 --- a/pkgs/community/swarmauri_community/vector_stores/MlmVectorStore.py +++ b/pkgs/community/swarmauri_community/vector_stores/MlmVectorStore.py @@ -1,6 +1,6 @@ from typing import List, Union, Literal from swarmauri.documents.concrete.Document import Document -from pkgs.community.swarmauri_community.embeddings.concrete.MlmEmbedding import MlmEmbedding +from swarmauri_community.embeddings.concrete.MlmEmbedding import MlmEmbedding from swarmauri.distances.concrete.CosineDistance import CosineDistance from swarmauri.vector_stores.base.VectorStoreBase import VectorStoreBase diff --git a/pkgs/community/tests/unit/embeddings/MlmEmbedding_unit_test.py b/pkgs/community/tests/unit/embeddings/MlmEmbedding_unit_test.py index 399318ebd..c015aeac3 100644 --- a/pkgs/community/tests/unit/embeddings/MlmEmbedding_unit_test.py +++ b/pkgs/community/tests/unit/embeddings/MlmEmbedding_unit_test.py @@ -1,5 +1,5 @@ import pytest -from pkgs.community.swarmauri_community.embeddings.concrete.MlmEmbedding import MlmEmbedding +from swarmauri_community.embeddings.concrete.MlmEmbedding import MlmEmbedding @pytest.mark.unit def test_ubc_resource(): diff --git a/pkgs/community/tests/unit/vector_stores/MlmVectorStore_unit_test.py b/pkgs/community/tests/unit/vector_stores/MlmVectorStore_unit_test.py index 27e300fe7..5f486aebb 100644 --- a/pkgs/community/tests/unit/vector_stores/MlmVectorStore_unit_test.py +++ b/pkgs/community/tests/unit/vector_stores/MlmVectorStore_unit_test.py @@ -1,6 +1,6 @@ import pytest from swarmauri.documents.concrete.Document import Document -from pkgs.community.swarmauri_community.vector_stores.MlmVectorStore import MlmVectorStore +from swarmauri_community.vector_stores.MlmVectorStore import MlmVectorStore @pytest.mark.unit From 719710a6224ebb0d9a55e28ca8b253642a6b0b80 Mon Sep 17 00:00:00 2001 From: michaeldecent2 <111002205+MichaelDecent@users.noreply.github.com> Date: Wed, 20 Nov 2024 13:50:18 +0100 Subject: [PATCH 009/168] swarm - reverted unfinished work --- .../swarmauri/tools/base/ToolBase.py | 3 - .../utils/extract_signature_decorator.py | 85 ------------------- 2 files changed, 88 deletions(-) delete mode 100644 pkgs/swarmauri/swarmauri/utils/extract_signature_decorator.py diff --git a/pkgs/swarmauri/swarmauri/tools/base/ToolBase.py b/pkgs/swarmauri/swarmauri/tools/base/ToolBase.py index dedd82244..3dfade766 100644 --- a/pkgs/swarmauri/swarmauri/tools/base/ToolBase.py +++ b/pkgs/swarmauri/swarmauri/tools/base/ToolBase.py @@ -19,9 +19,6 @@ def call(self, *args, **kwargs): @abstractmethod def __call__(self, *args, **kwargs): raise NotImplementedError("Subclasses must implement the __call__ method.") - - def extract_signature_details(self): - pass # #def __getstate__(self): diff --git a/pkgs/swarmauri/swarmauri/utils/extract_signature_decorator.py b/pkgs/swarmauri/swarmauri/utils/extract_signature_decorator.py deleted file mode 100644 index 412d10b90..000000000 --- a/pkgs/swarmauri/swarmauri/utils/extract_signature_decorator.py +++ /dev/null @@ -1,85 +0,0 @@ -from typing import ( - List, - Any, - Union, - Optional, - Callable, - get_type_hints, - get_args, - get_origin, -) -import inspect -from pydantic import BaseModel -from swarmauri.tools.concrete.Parameter import Parameter - - -class MethodSignatureExtractor(BaseModel): - parameters: List[Parameter] = [] - method: Callable - _type_mapping: dict = { - int: "integer", - float: "number", - str: "string", - bool: "boolean", - list: "array", - dict: "object", - Any: "any", - } - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.parameters = self.extract_signature_details() - - def _python_type_to_json_schema_type(self, py_type): - if get_origin(py_type) is not None: - origin = get_origin(py_type) - args = get_args(py_type) - - if origin is list: - items_type = self._python_type_to_json_schema_type(args[0]) - return {"type": "array", "items": items_type} - elif origin is dict: - return {"type": "object"} - elif origin in (Union, Optional): - if len(args) == 2 and type(None) in args: - non_none_type = args[0] if args[1] is type(None) else args[1] - return self._python_type_to_json_schema_type(non_none_type) - return { - "oneOf": [ - self._python_type_to_json_schema_type(arg) for arg in args - ] - } - return {"type": self._type_mapping.get(origin, "string")} - else: - return {"type": self._type_mapping.get(py_type, "string")} - - def extract_signature_details(self): - sig = inspect.signature(self.method) - type_hints = get_type_hints(self.method) - parameters = sig.parameters - details_list = [] - - for param_name, param in parameters.items(): - if param_name == "self": - continue - - param_type = type_hints.get(param_name, Any) - param_default = ( - param.default if param.default is not inspect.Parameter.empty else None - ) - required = param.default is inspect.Parameter.empty - enum = None - param_type_json_schema = self._python_type_to_json_schema_type(param_type) - - description = f"Parameter {param_name} of type {param_type_json_schema}" - - detail = Parameter( - name=param_name, - type=param_type_json_schema["type"], - description=description, - required=required, - enum=enum, - ) - details_list.append(detail) - - return details_list From ab4aca2c475543e96a1aafe589ee06e3c72605c7 Mon Sep 17 00:00:00 2001 From: 3rdSon Date: Wed, 20 Nov 2024 15:08:23 +0100 Subject: [PATCH 010/168] swarm - implemented hyperbolicvision --- .../llms/concrete/HyperbolicVisionModel.py | 378 ++++++++++++++++++ .../llms/HyperbolicVisionModel_unit_test.py | 158 ++++++++ 2 files changed, 536 insertions(+) create mode 100644 pkgs/swarmauri/swarmauri/llms/concrete/HyperbolicVisionModel.py create mode 100644 pkgs/swarmauri/tests/unit/llms/HyperbolicVisionModel_unit_test.py diff --git a/pkgs/swarmauri/swarmauri/llms/concrete/HyperbolicVisionModel.py b/pkgs/swarmauri/swarmauri/llms/concrete/HyperbolicVisionModel.py new file mode 100644 index 000000000..0dab82ce3 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/llms/concrete/HyperbolicVisionModel.py @@ -0,0 +1,378 @@ +import json +from pydantic import PrivateAttr +import httpx +from typing import List, Optional, Dict, Literal, Any, AsyncGenerator, Generator +import asyncio + +from swarmauri_core.typing import SubclassUnion +from swarmauri.conversations.concrete.Conversation import Conversation +from swarmauri.messages.base.MessageBase import MessageBase +from swarmauri.messages.concrete.AgentMessage import AgentMessage +from swarmauri.llms.base.LLMBase import LLMBase +from swarmauri.messages.concrete.AgentMessage import UsageData +from swarmauri.utils.retry_decorator import retry_on_status_codes +from swarmauri.utils.file_path_to_base64 import file_path_to_base64 + + +class HyperbolicVisionModel(LLMBase): + """ + HyperbolicVisionModel class for interacting with the Hyperbolic vision language models API. This class + provides synchronous and asynchronous methods to send conversation data to the + model, receive predictions, and stream responses. + + Attributes: + api_key (str): API key for authenticating requests to the Hyperbolic API. + allowed_models (List[str]): List of allowed model names that can be used. + name (str): The default model name to use for predictions. + type (Literal["HyperbolicVisionModel"]): The type identifier for this class. + """ + + api_key: str + allowed_models: List[str] = [ + "Qwen/Qwen2-VL-72B-Instruct", + "mistralai/Pixtral-12B-2409", + "Qwen/Qwen2-VL-7B-Instruct", + ] + name: str = "Qwen/Qwen2-VL-72B-Instruct" + type: Literal["HyperbolicVisionModel"] = "HyperbolicVisionModel" + _headers: Dict[str, str] = PrivateAttr(default=None) + _client: httpx.Client = PrivateAttr(default=None) + _BASE_URL: str = PrivateAttr( + default="https://api.hyperbolic.xyz/v1/chat/completions" + ) + + def __init__(self, **data): + """ + Initialize the HyperbolicVisionModel class with the provided data. + + Args: + **data: Arbitrary keyword arguments containing initialization data. + """ + super().__init__(**data) + self._headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {self.api_key}", + } + self._client = httpx.Client( + headers=self._headers, + base_url=self._BASE_URL, + ) + + def _format_messages( + self, + messages: List[SubclassUnion[MessageBase]], + ) -> List[Dict[str, Any]]: + """ + Formats conversation messages into the structure expected by the API. + + Args: + messages (List[MessageBase]): List of message objects from the conversation history. + + Returns: + List[Dict[str, Any]]: List of formatted message dictionaries. + """ + formatted_messages = [] + for message in messages: + formatted_message = message.model_dump( + include=["content", "role", "name"], exclude_none=True + ) + + if isinstance(formatted_message["content"], list): + formatted_content = [] + for item in formatted_message["content"]: + if item["type"] == "image_url" and "file_path" in item: + # Convert file path to base64 + base64_img = file_path_to_base64(item["file_path"]) + formatted_content.append( + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{base64_img}" + }, + } + ) + else: + formatted_content.append(item) + formatted_message["content"] = formatted_content + + formatted_messages.append(formatted_message) + return formatted_messages + + def _prepare_usage_data(self, usage_data) -> UsageData: + """ + Prepares and validates usage data received from the API response. + + Args: + usage_data (dict): Raw usage data from the API response. + + Returns: + UsageData: Validated usage data instance. + """ + return UsageData.model_validate(usage_data) + + @retry_on_status_codes((429, 529), max_retries=1) + def predict( + self, + conversation: Conversation, + temperature: float = 0.7, + max_tokens: int = 2048, + top_p: float = 0.9, + stop: Optional[List[str]] = None, + ) -> Conversation: + """ + Generates a response from the model based on the given conversation. + + Args: + conversation (Conversation): Conversation object with message history. + temperature (float): Sampling temperature for response diversity. + max_tokens (int): Maximum tokens for the model's response. + top_p (float): Cumulative probability for nucleus sampling. + stop (Optional[List[str]]): List of stop sequences for response termination. + + Returns: + Conversation: Updated conversation with the model's response. + """ + formatted_messages = self._format_messages(conversation.history) + payload = { + "model": self.name, + "messages": formatted_messages, + "temperature": temperature, + "max_tokens": max_tokens, + "top_p": top_p, + "stop": stop or [], + } + + response = self._client.post(self._BASE_URL, json=payload) + response.raise_for_status() + + response_data = response.json() + + message_content = response_data["choices"][0]["message"]["content"] + usage_data = response_data.get("usage", {}) + + usage = self._prepare_usage_data(usage_data) + conversation.add_message(AgentMessage(content=message_content, usage=usage)) + return conversation + + @retry_on_status_codes((429, 529), max_retries=1) + async def apredict( + self, + conversation: Conversation, + temperature: float = 0.7, + max_tokens: int = 2048, + top_p: float = 0.9, + stop: Optional[List[str]] = None, + ) -> Conversation: + """ + Async method to generate a response from the model based on the given conversation. + + Args: + conversation (Conversation): Conversation object with message history. + temperature (float): Sampling temperature for response diversity. + max_tokens (int): Maximum tokens for the model's response. + top_p (float): Cumulative probability for nucleus sampling. + stop (Optional[List[str]]): List of stop sequences for response termination. + + Returns: + Conversation: Updated conversation with the model's response. + """ + formatted_messages = self._format_messages(conversation.history) + payload = { + "model": self.name, + "messages": formatted_messages, + "temperature": temperature, + "max_tokens": max_tokens, + "top_p": top_p, + "stop": stop or [], + } + + async with httpx.AsyncClient() as async_client: + response = await async_client.post( + self._BASE_URL, json=payload, headers=self._headers + ) + response.raise_for_status() + + response_data = response.json() + + message_content = response_data["choices"][0]["message"]["content"] + usage_data = response_data.get("usage", {}) + + usage = self._prepare_usage_data(usage_data) + conversation.add_message(AgentMessage(content=message_content, usage=usage)) + return conversation + + @retry_on_status_codes((429, 529), max_retries=1) + def stream( + self, + conversation: Conversation, + temperature: float = 0.7, + max_tokens: int = 2048, + top_p: float = 0.9, + stop: Optional[List[str]] = None, + ) -> Generator[str, None, None]: + """ + Streams response text from the model in real-time. + + Args: + conversation (Conversation): Conversation object with message history. + temperature (float): Sampling temperature for response diversity. + max_tokens (int): Maximum tokens for the model's response. + top_p (float): Cumulative probability for nucleus sampling. + stop (Optional[List[str]]): List of stop sequences for response termination. + + Yields: + str: Partial response content from the model. + """ + formatted_messages = self._format_messages(conversation.history) + payload = { + "model": self.name, + "messages": formatted_messages, + "temperature": temperature, + "max_tokens": max_tokens, + "top_p": top_p, + "stream": True, + "stop": stop or [], + } + + response = self._client.post(self._BASE_URL, json=payload) + response.raise_for_status() + + message_content = "" + for line in response.iter_lines(): + json_str = line.replace("data: ", "") + try: + if json_str: + chunk = json.loads(json_str) + if chunk["choices"][0]["delta"]: + delta = chunk["choices"][0]["delta"]["content"] + message_content += delta + yield delta + except json.JSONDecodeError: + pass + + conversation.add_message(AgentMessage(content=message_content)) + + @retry_on_status_codes((429, 529), max_retries=1) + async def astream( + self, + conversation: Conversation, + temperature: float = 0.7, + max_tokens: int = 2048, + top_p: float = 0.9, + stop: Optional[List[str]] = None, + ) -> AsyncGenerator[str, None]: + """ + Async generator that streams response text from the model in real-time. + + Args: + conversation (Conversation): Conversation object with message history. + temperature (float): Sampling temperature for response diversity. + max_tokens (int): Maximum tokens for the model's response. + top_p (float): Cumulative probability for nucleus sampling. + stop (Optional[List[str]]): List of stop sequences for response termination. + + Yields: + str: Partial response content from the model. + """ + formatted_messages = self._format_messages(conversation.history) + payload = { + "model": self.name, + "messages": formatted_messages, + "temperature": temperature, + "max_tokens": max_tokens, + "top_p": top_p, + "stream": True, + "stop": stop or [], + } + + async with httpx.AsyncClient as async_client: + response = await async_client.post( + self._BASE_URL, json=payload, headers=self._headers + ) + response.raise_for_status() + + message_content = "" + async for line in response.aiter_lines(): + json_str = line.replace("data: ", "") + try: + if json_str: + chunk = json.loads(json_str) + if chunk["choices"][0]["delta"]: + delta = chunk["choices"][0]["delta"]["content"] + message_content += delta + yield delta + except json.JSONDecodeError: + pass + + conversation.add_message(AgentMessage(content=message_content)) + + def batch( + self, + conversations: List[Conversation], + temperature: float = 0.7, + max_tokens: int = 2048, + top_p: float = 0.9, + stop: Optional[List[str]] = None, + ) -> List[Conversation]: + """ + Processes a batch of conversations and generates responses for each sequentially. + + Args: + conversations (List[Conversation]): List of conversations to process. + temperature (float): Sampling temperature for response diversity. + max_tokens (int): Maximum tokens for each response. + top_p (float): Cumulative probability for nucleus sampling. + stop (Optional[List[str]]): List of stop sequences for response termination. + + Returns: + List[Conversation]: List of updated conversations with model responses. + """ + results = [] + for conversation in conversations: + result_conversation = self.predict( + conversation, + temperature=temperature, + max_tokens=max_tokens, + top_p=top_p, + stop=stop, + ) + results.append(result_conversation) + return results + + async def abatch( + self, + conversations: List[Conversation], + temperature: float = 0.7, + max_tokens: int = 2048, + top_p: float = 0.9, + stop: Optional[List[str]] = None, + max_concurrent=5, + ) -> List[Conversation]: + """ + Async method for processing a batch of conversations concurrently. + + Args: + conversations (List[Conversation]): List of conversations to process. + temperature (float): Sampling temperature for response diversity. + max_tokens (int): Maximum tokens for each response. + top_p (float): Cumulative probability for nucleus sampling. + stop (Optional[List[str]]): List of stop sequences for response termination. + max_concurrent (int): Maximum number of concurrent requests. + + Returns: + List[Conversation]: List of updated conversations with model responses. + """ + semaphore = asyncio.Semaphore(max_concurrent) + + async def process_conversation(conv: Conversation) -> Conversation: + async with semaphore: + return await self.apredict( + conv, + temperature=temperature, + max_tokens=max_tokens, + top_p=top_p, + stop=stop, + ) + + tasks = [process_conversation(conv) for conv in conversations] + return await asyncio.gather(*tasks) diff --git a/pkgs/swarmauri/tests/unit/llms/HyperbolicVisionModel_unit_test.py b/pkgs/swarmauri/tests/unit/llms/HyperbolicVisionModel_unit_test.py new file mode 100644 index 000000000..495341aae --- /dev/null +++ b/pkgs/swarmauri/tests/unit/llms/HyperbolicVisionModel_unit_test.py @@ -0,0 +1,158 @@ +import pytest +import os +from swarmauri.llms.concrete.HyperbolicVisionModel import HyperbolicVisionModel +from swarmauri.conversations.concrete.Conversation import Conversation +from swarmauri.messages.concrete.HumanMessage import HumanMessage +from dotenv import load_dotenv +from swarmauri.utils.timeout_wrapper import timeout + +load_dotenv() + +API_KEY = os.getenv("HYPERBOLIC_API_KEY") + + +@pytest.fixture(scope="module") +def hyperbolic_vision_model(): + if not API_KEY: + pytest.skip("Skipping due to environment variable not set") + model = HyperbolicVisionModel(api_key=API_KEY) + return model + + +def get_allowed_models(): + if not API_KEY: + return [] + model = HyperbolicVisionModel(api_key=API_KEY) + return model.allowed_models + + +@timeout(5) +@pytest.mark.unit +def test_ubc_resource(hyperbolic_vision_model): + assert hyperbolic_vision_model.resource == "LLM" + + +@timeout(5) +@pytest.mark.unit +def test_ubc_type(hyperbolic_vision_model): + assert hyperbolic_vision_model.type == "HyperbolicVisionModel" + + +@timeout(5) +@pytest.mark.unit +def test_serialization(hyperbolic_vision_model): + assert ( + hyperbolic_vision_model.id + == HyperbolicVisionModel.model_validate_json( + hyperbolic_vision_model.model_dump_json() + ).id + ) + + +@timeout(5) +@pytest.mark.unit +def test_default_model_name(hyperbolic_vision_model): + assert hyperbolic_vision_model.name == "Qwen/Qwen2-VL-72B-Instruct" + + +def create_test_conversation(image_url, prompt): + conversation = Conversation() + conversation.add_message( + HumanMessage( + content=[ + {"type": "text", "text": prompt}, + {"type": "image_url", "image_url": {"url": image_url}}, + ] + ) + ) + return conversation + + +@pytest.mark.parametrize("model_name", get_allowed_models()) +@timeout(5) +@pytest.mark.unit +def test_predict(hyperbolic_vision_model, model_name): + model = hyperbolic_vision_model + model.name = model_name + + image_url = "https://llava-vl.github.io/static/images/monalisa.jpg" + prompt = "Who painted this artwork?" + conversation = create_test_conversation(image_url, prompt) + + result = model.predict(conversation) + + assert result.history[-1].content is not None + assert isinstance(result.history[-1].content, str) + assert len(result.history[-1].content) > 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_name", get_allowed_models()) +@timeout(5) +@pytest.mark.unit +async def test_apredict(hyperbolic_vision_model, model_name): + model = hyperbolic_vision_model + model.name = model_name + + image_url = "https://llava-vl.github.io/static/images/monalisa.jpg" + prompt = "Describe the woman in the painting." + conversation = create_test_conversation(image_url, prompt) + + result = await model.apredict(conversation) + + assert result.history[-1].content is not None + assert isinstance(result.history[-1].content, str) + assert len(result.history[-1].content) > 0 + + +@timeout(5) +@pytest.mark.unit +def test_batch(hyperbolic_vision_model): + image_urls = [ + "https://llava-vl.github.io/static/images/monalisa.jpg", + "https://llava-vl.github.io/static/images/monalisa.jpg", + ] + prompts = [ + "Who painted this artwork?", + "Describe the woman in the painting.", + ] + + conversations = [ + create_test_conversation(image_url, prompt) + for image_url, prompt in zip(image_urls, prompts) + ] + + results = hyperbolic_vision_model.batch(conversations) + + assert len(results) == len(image_urls) + for result in results: + assert result.history[-1].content is not None + assert isinstance(result.history[-1].content, str) + assert len(result.history[-1].content) > 0 + + +@pytest.mark.asyncio +@timeout(5) +@pytest.mark.unit +async def test_abatch(hyperbolic_vision_model): + image_urls = [ + "https://llava-vl.github.io/static/images/monalisa.jpg", + "https://llava-vl.github.io/static/images/monalisa.jpg", + ] + prompts = [ + "Who painted this artwork?", + "Describe the woman in the painting.", + ] + + conversations = [ + create_test_conversation(image_url, prompt) + for image_url, prompt in zip(image_urls, prompts) + ] + + results = await hyperbolic_vision_model.abatch(conversations) + + assert len(results) == len(image_urls) + for result in results: + assert result.history[-1].content is not None + assert isinstance(result.history[-1].content, str) + assert len(result.history[-1].content) > 0 From a17005800b6411640bba6f7bae1a3358c500b1ef Mon Sep 17 00:00:00 2001 From: 3rdSon Date: Wed, 20 Nov 2024 15:11:45 +0100 Subject: [PATCH 011/168] swarm - hyperbolic --- .../swarmauri/swarmauri/llms/concrete/HyperbolicVisionModel.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkgs/swarmauri/swarmauri/llms/concrete/HyperbolicVisionModel.py b/pkgs/swarmauri/swarmauri/llms/concrete/HyperbolicVisionModel.py index 0dab82ce3..14e2d196a 100644 --- a/pkgs/swarmauri/swarmauri/llms/concrete/HyperbolicVisionModel.py +++ b/pkgs/swarmauri/swarmauri/llms/concrete/HyperbolicVisionModel.py @@ -25,6 +25,9 @@ class HyperbolicVisionModel(LLMBase): allowed_models (List[str]): List of allowed model names that can be used. name (str): The default model name to use for predictions. type (Literal["HyperbolicVisionModel"]): The type identifier for this class. + + Link to Allowed Models: https://app.hyperbolic.xyz/models + Link to API KEYS: https://app.hyperbolic.xyz/settings """ api_key: str From 4421b1a293cf4a706e0020c9cfafbcac552287c8 Mon Sep 17 00:00:00 2001 From: michaeldecent2 <111002205+MichaelDecent@users.noreply.github.com> Date: Wed, 20 Nov 2024 15:55:39 +0100 Subject: [PATCH 012/168] comm - refactor: update dependencies and add optional extras in pyproject.toml --- pkgs/community/pyproject.toml | 87 +++++++++++++++++++---------------- 1 file changed, 48 insertions(+), 39 deletions(-) diff --git a/pkgs/community/pyproject.toml b/pkgs/community/pyproject.toml index b9b8d8665..3d60c38ad 100644 --- a/pkgs/community/pyproject.toml +++ b/pkgs/community/pyproject.toml @@ -15,48 +15,59 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.13" -captcha = "*" -chromadb = "*" -duckdb = "*" -folium = "*" -gensim = "*" -#google-generativeai = "*" -gradio = "*" -leptonai = "0.22.0" -neo4j = "*" -nltk = "*" -#openai = "^1.52.0" -pandas = "*" -psutil = "*" -pygithub = "*" -python-dotenv = "*" -qrcode = "*" -redis = "^4.0" -#scikit-learn="^1.4.2" swarmauri = "==0.5.2" -textstat = "*" -transformers = ">=4.45.0" typing_extensions = "*" -tiktoken = "*" -pymupdf = "*" -annoy = "*" -qdrant_client = "*" -weaviate = "*" -pinecone-client = { version = "*", extras = ["grpc"] } -PyPDF2 = "*" -pypdftk = "*" -weaviate-client = "*" -protobuf = "^3.20.0" -# Pacmap requires specific version of numba -#numba = ">=0.59.0" -#pacmap = "==0.7.3" +matplotlib = { version = ">=3.9.2", optional = true } +nltk = { version = "^3.9.1", optional = true } +gensim = { version = "==4.3.3", optional = true } +transformers = { version = "^4.45.0", optional = true } +spacy = { version = ">=3.0.0,<=3.8.2", optional = true } +textblob = { version = "^0.18.0", optional = true } +torch = { version = "^2.5.0", optional = true } +leptonai = { version = "==0.22.0", optional = true } +redis = { version = "^4.0", optional = true } +#protobuf = { version = "^3.20.0", optional = true } +#numba = { version = ">=0.59.0", optional = true } +#pacmap = { version = "==0.7.3", optional = true } +[tool.poetry.extras] +# Grouped optional dependencies +nlp = ["nltk", "textblob", "textstat", "gensim"] +ml_toolkits = ["transformers", "annoy"] +visualization = ["folium", "matplotlib"] +storage = ["redis", "duckdb", "neo4j", "chromadb", "qdrant_client", "weaviate-client", "pinecone-client[grpc]"] +document_processing = ["PyPDF2", "pymupdf", "pypdftk"] +cloud_integration = ["psutil", "qrcode", "pygithub"] +spacy = ["spacy"] +transformers = ["transformers"] +torch = ["torch"] +gradio = ["gradio"] +model_clients = ["leptonai", "google-generativeai", "openai"] +tiktoken = ["tiktoken"] +#protobuf = ["protobuf"] +#pacmap = ["numba", "pacmap"] + +# Full installation +full = [ + "nltk", "gensim", "textstat", + "transformers", "annoy", + "folium", "matplotlib", + "redis", "duckdb", "neo4j", "chromadb", "qdrant_client", "weaviate-client", "pinecone-client[grpc]", + "PyPDF2", "pymupdf", "pypdftk", + "psutil", "qrcode", "pygithub", + "scipy", "spacy", + "torch", + "gradio", + "leptonai", "google-generativeai", "openai" + #"pacmap", "numba" +] [tool.poetry.dev-dependencies] -flake8 = "^7.0" # Add flake8 as a development dependency -pytest = "^8.0" # Ensure pytest is also added if you run tests +flake8 = "^7.0" +pytest = "^8.0" pytest-asyncio = ">=0.24.0" pytest-xdist = "^3.6.1" +python-dotenv = "^1.0.0" [build-system] requires = ["poetry-core>=1.0.0"] @@ -70,12 +81,10 @@ markers = [ "unit: Unit tests", "integration: Integration tests", "acceptance: Acceptance tests", - "experimental: Experimental tests", + "experimental: Experimental tests" ] - log_cli = true log_cli_level = "INFO" log_cli_format = "%(asctime)s [%(levelname)s] %(message)s" log_cli_date_format = "%Y-%m-%d %H:%M:%S" - -asyncio_default_fixture_loop_scope = "function" +asyncio_default_fixture_loop_scope = "function" \ No newline at end of file From 2b05d476ee825bd1b7f21112392a5dc64717434c Mon Sep 17 00:00:00 2001 From: michaeldecent2 <111002205+MichaelDecent@users.noreply.github.com> Date: Wed, 20 Nov 2024 16:21:14 +0100 Subject: [PATCH 013/168] comm - minor fix --- pkgs/community/pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkgs/community/pyproject.toml b/pkgs/community/pyproject.toml index 3d60c38ad..332deb6bd 100644 --- a/pkgs/community/pyproject.toml +++ b/pkgs/community/pyproject.toml @@ -26,6 +26,7 @@ textblob = { version = "^0.18.0", optional = true } torch = { version = "^2.5.0", optional = true } leptonai = { version = "==0.22.0", optional = true } redis = { version = "^4.0", optional = true } +pinecone-client = { version = ">=2.0.0", optional = true, extras = ["grpc"] } #protobuf = { version = "^3.20.0", optional = true } #numba = { version = ">=0.59.0", optional = true } #pacmap = { version = "==0.7.3", optional = true } @@ -35,7 +36,7 @@ redis = { version = "^4.0", optional = true } nlp = ["nltk", "textblob", "textstat", "gensim"] ml_toolkits = ["transformers", "annoy"] visualization = ["folium", "matplotlib"] -storage = ["redis", "duckdb", "neo4j", "chromadb", "qdrant_client", "weaviate-client", "pinecone-client[grpc]"] +storage = ["redis", "duckdb", "neo4j", "chromadb", "qdrant_client", "weaviate-client", "pinecone-client"] document_processing = ["PyPDF2", "pymupdf", "pypdftk"] cloud_integration = ["psutil", "qrcode", "pygithub"] spacy = ["spacy"] @@ -52,7 +53,7 @@ full = [ "nltk", "gensim", "textstat", "transformers", "annoy", "folium", "matplotlib", - "redis", "duckdb", "neo4j", "chromadb", "qdrant_client", "weaviate-client", "pinecone-client[grpc]", + "redis", "duckdb", "neo4j", "chromadb", "qdrant_client", "weaviate-client", "pinecone-client", "PyPDF2", "pymupdf", "pypdftk", "psutil", "qrcode", "pygithub", "scipy", "spacy", From feecdc1dcc3c9c0aeec3204949e3b5d70cbb0fe1 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Wed, 20 Nov 2024 16:19:56 -0600 Subject: [PATCH 014/168] Update ImageGenBase.py --- pkgs/swarmauri/swarmauri/image_gens/base/ImageGenBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/swarmauri/image_gens/base/ImageGenBase.py b/pkgs/swarmauri/swarmauri/image_gens/base/ImageGenBase.py index e9fbda41e..ab3dff1f2 100644 --- a/pkgs/swarmauri/swarmauri/image_gens/base/ImageGenBase.py +++ b/pkgs/swarmauri/swarmauri/image_gens/base/ImageGenBase.py @@ -9,7 +9,7 @@ class ImageGenBase(IGenImage, ComponentBase): allowed_models: List[str] = [] resource: Optional[str] = Field(default=ResourceTypes.IMAGE_GEN.value, frozen=True) model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) - type: Literal["LLMBase"] = "LLMBase" + type: Literal["ImageGenBase"] = "ImageGenBase" @model_validator(mode="after") @classmethod From 2147e9265f51108e98078726f2184c8b4960fa7c Mon Sep 17 00:00:00 2001 From: michaeldecent2 <111002205+MichaelDecent@users.noreply.github.com> Date: Thu, 21 Nov 2024 05:29:02 +0100 Subject: [PATCH 015/168] swarm - moved image gen models --- .../concrete/BlackForestImgGenModel.py | 7 +- .../concrete/DeepInfraImgGenModel.py | 8 +- .../concrete/FalAIImgGenModel.py | 8 +- .../concrete/HyperbolicImgGenModel.py | 12 +- .../concrete/OpenAIImgGenModel.py | 8 +- .../llms/concrete/BlackForestImgGenModel.py | 259 ------------------ .../DeepInfraImgGenModel_unit_test.py | 2 +- .../FalAIImgGenModel_unit_test.py | 2 +- .../HyperbolicImgGenModel_unit_test.py | 2 +- .../OpenAIImgGenModel_unit_tesst.py | 2 +- .../llms/BlackForestImgGenModel_unit_test.py | 120 -------- 11 files changed, 25 insertions(+), 405 deletions(-) rename pkgs/swarmauri/swarmauri/{llms => image_gens}/concrete/DeepInfraImgGenModel.py (97%) rename pkgs/swarmauri/swarmauri/{llms => image_gens}/concrete/FalAIImgGenModel.py (98%) rename pkgs/swarmauri/swarmauri/{llms => image_gens}/concrete/HyperbolicImgGenModel.py (96%) rename pkgs/swarmauri/swarmauri/{llms => image_gens}/concrete/OpenAIImgGenModel.py (97%) delete mode 100644 pkgs/swarmauri/swarmauri/llms/concrete/BlackForestImgGenModel.py rename pkgs/swarmauri/tests/unit/{llms => image_gens}/DeepInfraImgGenModel_unit_test.py (97%) rename pkgs/swarmauri/tests/unit/{llms => image_gens}/FalAIImgGenModel_unit_test.py (97%) rename pkgs/swarmauri/tests/unit/{llms => image_gens}/HyperbolicImgGenModel_unit_test.py (97%) rename pkgs/swarmauri/tests/unit/{llms => image_gens}/OpenAIImgGenModel_unit_tesst.py (97%) delete mode 100644 pkgs/swarmauri/tests/unit/llms/BlackForestImgGenModel_unit_test.py diff --git a/pkgs/swarmauri/swarmauri/image_gens/concrete/BlackForestImgGenModel.py b/pkgs/swarmauri/swarmauri/image_gens/concrete/BlackForestImgGenModel.py index c697764ff..202d5c631 100644 --- a/pkgs/swarmauri/swarmauri/image_gens/concrete/BlackForestImgGenModel.py +++ b/pkgs/swarmauri/swarmauri/image_gens/concrete/BlackForestImgGenModel.py @@ -1,6 +1,6 @@ import httpx import time -from typing import List, Literal, Optional, Dict, ClassVar +from typing import List, Literal, Optional, Dict from pydantic import PrivateAttr from swarmauri.utils.retry_decorator import retry_on_status_codes from swarmauri.image_gens.base.ImageGenBase import ImageGenBase @@ -21,15 +21,14 @@ class BlackForestImgGenModel(ImageGenBase): api_key: str allowed_models: List[str] = ["flux-pro-1.1", "flux-pro", "flux-dev"] - asyncio: ClassVar = asyncio name: str = "flux-pro" # Default model type: Literal["BlackForestImgGenModel"] = "BlackForestImgGenModel" - def __init__(self, **data): + def __init__(self, **kwargs): """ Initializes the BlackForestImgGenModel instance with HTTP clients. """ - super().__init__(**data) + super().__init__(**kwargs) self._headers = { "Content-Type": "application/json", "X-Key": self.api_key, diff --git a/pkgs/swarmauri/swarmauri/llms/concrete/DeepInfraImgGenModel.py b/pkgs/swarmauri/swarmauri/image_gens/concrete/DeepInfraImgGenModel.py similarity index 97% rename from pkgs/swarmauri/swarmauri/llms/concrete/DeepInfraImgGenModel.py rename to pkgs/swarmauri/swarmauri/image_gens/concrete/DeepInfraImgGenModel.py index 56afc2105..5dcc35d98 100644 --- a/pkgs/swarmauri/swarmauri/llms/concrete/DeepInfraImgGenModel.py +++ b/pkgs/swarmauri/swarmauri/image_gens/concrete/DeepInfraImgGenModel.py @@ -2,12 +2,12 @@ from typing import List, Literal from pydantic import PrivateAttr from swarmauri.utils.retry_decorator import retry_on_status_codes -from swarmauri.llms.base.LLMBase import LLMBase +from swarmauri.image_gens.base.ImageGenBase import ImageGenBase import asyncio import contextlib -class DeepInfraImgGenModel(LLMBase): +class DeepInfraImgGenModel(ImageGenBase): """ A model class for generating images from text prompts using DeepInfra's image generation API. @@ -37,7 +37,7 @@ class DeepInfraImgGenModel(LLMBase): name: str = "stabilityai/stable-diffusion-2-1" # Default model type: Literal["DeepInfraImgGenModel"] = "DeepInfraImgGenModel" - def __init__(self, **data): + def __init__(self, **kwargs): """ Initializes the DeepInfraImgGenModel instance. @@ -47,7 +47,7 @@ def __init__(self, **data): Args: **data: Keyword arguments for model initialization. """ - super().__init__(**data) + super().__init__(**kwargs) self._headers = { "Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}", diff --git a/pkgs/swarmauri/swarmauri/llms/concrete/FalAIImgGenModel.py b/pkgs/swarmauri/swarmauri/image_gens/concrete/FalAIImgGenModel.py similarity index 98% rename from pkgs/swarmauri/swarmauri/llms/concrete/FalAIImgGenModel.py rename to pkgs/swarmauri/swarmauri/image_gens/concrete/FalAIImgGenModel.py index 6943d1e59..b6eadb3b7 100644 --- a/pkgs/swarmauri/swarmauri/llms/concrete/FalAIImgGenModel.py +++ b/pkgs/swarmauri/swarmauri/image_gens/concrete/FalAIImgGenModel.py @@ -3,11 +3,11 @@ from typing import List, Literal, Optional, Dict from pydantic import Field, PrivateAttr from swarmauri.utils.retry_decorator import retry_on_status_codes -from swarmauri.llms.base.LLMBase import LLMBase +from swarmauri.image_gens.base.ImageGenBase import ImageGenBase import time -class FalAIImgGenModel(LLMBase): +class FalAIImgGenModel(ImageGenBase): """ A model class for generating images from text using FluxPro's image generation model, provided by FalAI. This class uses a queue-based API to handle image generation requests. @@ -34,7 +34,7 @@ class FalAIImgGenModel(LLMBase): max_retries: int = Field(default=60) # Maximum number of status check retries retry_delay: float = Field(default=1.0) # Delay between status checks in seconds - def __init__(self, **data): + def __init__(self, **kwargs): """ Initializes the model with the specified API key and model name. @@ -44,7 +44,7 @@ def __init__(self, **data): Raises: ValueError: If an invalid model name is provided. """ - super().__init__(**data) + super().__init__(**kwargs) self._headers = { "Content-Type": "application/json", "Authorization": f"Key {self.api_key}", diff --git a/pkgs/swarmauri/swarmauri/llms/concrete/HyperbolicImgGenModel.py b/pkgs/swarmauri/swarmauri/image_gens/concrete/HyperbolicImgGenModel.py similarity index 96% rename from pkgs/swarmauri/swarmauri/llms/concrete/HyperbolicImgGenModel.py rename to pkgs/swarmauri/swarmauri/image_gens/concrete/HyperbolicImgGenModel.py index 72d6e8267..43f7dd60b 100644 --- a/pkgs/swarmauri/swarmauri/llms/concrete/HyperbolicImgGenModel.py +++ b/pkgs/swarmauri/swarmauri/image_gens/concrete/HyperbolicImgGenModel.py @@ -1,13 +1,13 @@ import httpx -from typing import List, Literal, Optional +from typing import List, Literal from pydantic import PrivateAttr from swarmauri.utils.retry_decorator import retry_on_status_codes -from swarmauri.llms.base.LLMBase import LLMBase +from swarmauri.image_gens.base.ImageGenBase import ImageGenBase import asyncio import contextlib -class HyperbolicImgGenModel(LLMBase): +class HyperbolicImgGenModel(ImageGenBase): """ A model class for generating images from text prompts using Hyperbolic's image generation API. @@ -29,7 +29,7 @@ class HyperbolicImgGenModel(LLMBase): """ _BASE_URL: str = PrivateAttr("https://api.hyperbolic.xyz/v1/image/generation") - _client: httpx.Client = PrivateAttr() + _client: httpx.Client = PrivateAttr(default=None) _async_client: httpx.AsyncClient = PrivateAttr(default=None) api_key: str @@ -52,7 +52,7 @@ class HyperbolicImgGenModel(LLMBase): enable_refiner: bool = False backend: str = "auto" - def __init__(self, **data): + def __init__(self, **kwargs): """ Initializes the HyperbolicImgGenModel instance. @@ -62,7 +62,7 @@ def __init__(self, **data): Args: **data: Keyword arguments for model initialization. """ - super().__init__(**data) + super().__init__(**kwargs) self._headers = { "Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}", diff --git a/pkgs/swarmauri/swarmauri/llms/concrete/OpenAIImgGenModel.py b/pkgs/swarmauri/swarmauri/image_gens/concrete/OpenAIImgGenModel.py similarity index 97% rename from pkgs/swarmauri/swarmauri/llms/concrete/OpenAIImgGenModel.py rename to pkgs/swarmauri/swarmauri/image_gens/concrete/OpenAIImgGenModel.py index ad78fd7d8..8862799e5 100644 --- a/pkgs/swarmauri/swarmauri/llms/concrete/OpenAIImgGenModel.py +++ b/pkgs/swarmauri/swarmauri/image_gens/concrete/OpenAIImgGenModel.py @@ -2,11 +2,11 @@ import asyncio import httpx from typing import Dict, List, Literal, Optional +from swarmauri.image_gens.base.ImageGenBase import ImageGenBase from swarmauri.utils.retry_decorator import retry_on_status_codes -from swarmauri.llms.base.LLMBase import LLMBase -class OpenAIImgGenModel(LLMBase): +class OpenAIImgGenModel(ImageGenBase): """ OpenAIImgGenModel is a class for generating images using OpenAI's DALL-E models. @@ -26,14 +26,14 @@ class OpenAIImgGenModel(LLMBase): _BASE_URL: str = PrivateAttr(default="https://api.openai.com/v1/images/generations") _headers: Dict[str, str] = PrivateAttr(default=None) - def __init__(self, **data) -> None: + def __init__(self, **kwargs) -> None: """ Initialize the GroqAIAudio class with the provided data. Args: **data: Arbitrary keyword arguments containing initialization data. """ - super().__init__(**data) + super().__init__(**kwargs) self._headers = { "Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json", diff --git a/pkgs/swarmauri/swarmauri/llms/concrete/BlackForestImgGenModel.py b/pkgs/swarmauri/swarmauri/llms/concrete/BlackForestImgGenModel.py deleted file mode 100644 index 50d395394..000000000 --- a/pkgs/swarmauri/swarmauri/llms/concrete/BlackForestImgGenModel.py +++ /dev/null @@ -1,259 +0,0 @@ -import httpx -import time -from typing import List, Literal, Optional, Dict, ClassVar -from pydantic import PrivateAttr -from swarmauri.utils.retry_decorator import retry_on_status_codes -from swarmauri.llms.base.LLMBase import LLMBase -import asyncio -import contextlib - - -class BlackForestImgGenModel(LLMBase): - """ - A model for generating images using FluxPro's image generation models through the Black Forest API. - Link to API key: https://api.bfl.ml/auth/profile - """ - - _BASE_URL: str = PrivateAttr("https://api.bfl.ml") - _client: httpx.Client = PrivateAttr() - _async_client: httpx.AsyncClient = PrivateAttr(default=None) - - api_key: str - allowed_models: List[str] = ["flux-pro-1.1", "flux-pro", "flux-dev"] - - asyncio: ClassVar = asyncio - name: str = "flux-pro" # Default model - type: Literal["BlackForestImgGenModel"] = "BlackForestImgGenModel" - - def __init__(self, **data): - """ - Initializes the BlackForestImgGenModel instance with HTTP clients. - """ - super().__init__(**data) - self._headers = { - "Content-Type": "application/json", - "X-Key": self.api_key, - } - self._client = httpx.Client(headers=self._headers, timeout=30) - - async def _get_async_client(self) -> httpx.AsyncClient: - """Gets or creates an async client instance.""" - if self._async_client is None or self._async_client.is_closed: - self._async_client = httpx.AsyncClient(headers=self._headers, timeout=30) - return self._async_client - - async def _close_async_client(self): - """Closes the async client if it exists and is open.""" - if self._async_client is not None and not self._async_client.is_closed: - await self._async_client.aclose() - self._async_client = None - - @retry_on_status_codes((429, 529), max_retries=1) - def _send_request(self, endpoint: str, data: dict) -> dict: - """Send a synchronous request to FluxPro's API for image generation.""" - url = f"{self._BASE_URL}/{endpoint}" - response = self._client.post(url, json=data) - response.raise_for_status() - return response.json() - - @retry_on_status_codes((429, 529), max_retries=1) - async def _async_send_request(self, endpoint: str, data: dict) -> dict: - """Send an asynchronous request to FluxPro's API for image generation.""" - client = await self._get_async_client() - url = f"{self._BASE_URL}/{endpoint}" - response = await client.post(url, json=data) - response.raise_for_status() - return response.json() - - @retry_on_status_codes((429, 529), max_retries=1) - def _get_result(self, task_id: str) -> dict: - """Get the result of a generation task synchronously.""" - url = f"{self._BASE_URL}/v1/get_result" - params = {"id": task_id} - response = self._client.get(url, params=params) - response.raise_for_status() - return response.json() - - @retry_on_status_codes((429, 529), max_retries=1) - async def _async_get_result(self, task_id: str) -> dict: - """Get the result of a generation task asynchronously.""" - client = await self._get_async_client() - url = f"{self._BASE_URL}/v1/get_result" - params = {"id": task_id} - response = await client.get(url, params=params) - response.raise_for_status() - return response.json() - - def generate_image( - self, - prompt: str, - width: int = 1024, - height: int = 768, - steps: Optional[int] = None, - prompt_upsampling: bool = False, - seed: Optional[int] = None, - guidance: Optional[float] = None, - safety_tolerance: Optional[int] = None, - interval: Optional[float] = None, - max_wait_time: int = 300, - check_interval: int = 10, - ) -> Dict: - """ - Generates an image based on the prompt and waits for the result synchronously. - - Args: - prompt (str): The text prompt for image generation - width (int): Image width in pixels - height (int): Image height in pixels - steps (Optional[int]): Number of inference steps - prompt_upsampling (bool): Whether to use prompt upsampling - seed (Optional[int]): Random seed for generation - guidance (Optional[float]): Guidance scale - safety_tolerance (Optional[int]): Safety tolerance level - interval (Optional[float]): Interval parameter (flux-pro only) - max_wait_time (int): Maximum time to wait for result in seconds - check_interval (int): Time between status checks in seconds - - Returns: - Dict: Dictionary containing the image URL and other result information - """ - endpoint = f"v1/{self.name}" - data = { - "prompt": prompt, - "width": width, - "height": height, - "prompt_upsampling": prompt_upsampling, - } - - if steps is not None: - data["steps"] = steps - if seed is not None: - data["seed"] = seed - if guidance is not None: - data["guidance"] = guidance - if safety_tolerance is not None: - data["safety_tolerance"] = safety_tolerance - if interval is not None and self.name == "flux-pro": - data["interval"] = interval - - response = self._send_request(endpoint, data) - task_id = response["id"] - - start_time = time.time() - while time.time() - start_time < max_wait_time: - result = self._get_result(task_id) - if result["status"] == "Ready": - return result["result"]["sample"] - elif result["status"] in [ - "Error", - "Request Moderated", - "Content Moderated", - ]: - raise Exception(f"Task failed with status: {result['status']}") - time.sleep(check_interval) - - raise TimeoutError(f"Image generation timed out after {max_wait_time} seconds") - - async def agenerate_image(self, prompt: str, **kwargs) -> Dict: - """ - Asynchronously generates an image based on the prompt and waits for the result. - - Args: - prompt (str): The text prompt for image generation - **kwargs: Additional arguments passed to generate_image - - Returns: - Dict: Dictionary containing the image URL and other result information - """ - try: - endpoint = f"v1/{self.name}" - data = { - "prompt": prompt, - "width": kwargs.get("width", 1024), - "height": kwargs.get("height", 768), - "prompt_upsampling": kwargs.get("prompt_upsampling", False), - } - - optional_params = [ - "steps", - "seed", - "guidance", - "safety_tolerance", - ] - for param in optional_params: - if param in kwargs: - data[param] = kwargs[param] - - if "interval" in kwargs and self.name == "flux-pro": - data["interval"] = kwargs["interval"] - - response = await self._async_send_request(endpoint, data) - task_id = response["id"] - - max_wait_time = kwargs.get("max_wait_time", 300) - check_interval = kwargs.get("check_interval", 10) - start_time = time.time() - - while time.time() - start_time < max_wait_time: - result = await self._async_get_result(task_id) - if result["status"] == "Ready": - return result["result"]["sample"] - elif result["status"] in [ - "Error", - "Request Moderated", - "Content Moderated", - ]: - raise Exception(f"Task failed with status: {result['status']}") - await asyncio.sleep(check_interval) - - raise TimeoutError( - f"Image generation timed out after {max_wait_time} seconds" - ) - finally: - await self._close_async_client() - - def batch_generate(self, prompts: List[str], **kwargs) -> List[Dict]: - """ - Generates images for a batch of prompts synchronously. - - Args: - prompts (List[str]): List of text prompts - **kwargs: Additional arguments passed to generate_image - - Returns: - List[Dict]: List of result dictionaries - """ - return [self.generate_image(prompt=prompt, **kwargs) for prompt in prompts] - - async def abatch_generate( - self, prompts: List[str], max_concurrent: int = 5, **kwargs - ) -> List[Dict]: - """ - Asynchronously generates images for a batch of prompts. - - Args: - prompts (List[str]): List of text prompts - max_concurrent (int): Maximum number of concurrent tasks - **kwargs: Additional arguments passed to agenerate_image - - Returns: - List[Dict]: List of result dictionaries - """ - try: - semaphore = asyncio.Semaphore(max_concurrent) - - async def process_prompt(prompt): - async with semaphore: - return await self.agenerate_image(prompt=prompt, **kwargs) - - tasks = [process_prompt(prompt) for prompt in prompts] - return await asyncio.gather(*tasks) - finally: - await self._close_async_client() - - def __del__(self): - """Cleanup method to ensure clients are closed.""" - self._client.close() - if self._async_client is not None and not self._async_client.is_closed: - with contextlib.suppress(Exception): - asyncio.run(self._close_async_client()) diff --git a/pkgs/swarmauri/tests/unit/llms/DeepInfraImgGenModel_unit_test.py b/pkgs/swarmauri/tests/unit/image_gens/DeepInfraImgGenModel_unit_test.py similarity index 97% rename from pkgs/swarmauri/tests/unit/llms/DeepInfraImgGenModel_unit_test.py rename to pkgs/swarmauri/tests/unit/image_gens/DeepInfraImgGenModel_unit_test.py index 98b3b7047..5492ff573 100644 --- a/pkgs/swarmauri/tests/unit/llms/DeepInfraImgGenModel_unit_test.py +++ b/pkgs/swarmauri/tests/unit/image_gens/DeepInfraImgGenModel_unit_test.py @@ -1,6 +1,6 @@ import pytest import os -from swarmauri.llms.concrete.DeepInfraImgGenModel import DeepInfraImgGenModel +from swarmauri.image_gens.concrete.DeepInfraImgGenModel import DeepInfraImgGenModel from dotenv import load_dotenv from swarmauri.utils.timeout_wrapper import timeout diff --git a/pkgs/swarmauri/tests/unit/llms/FalAIImgGenModel_unit_test.py b/pkgs/swarmauri/tests/unit/image_gens/FalAIImgGenModel_unit_test.py similarity index 97% rename from pkgs/swarmauri/tests/unit/llms/FalAIImgGenModel_unit_test.py rename to pkgs/swarmauri/tests/unit/image_gens/FalAIImgGenModel_unit_test.py index bf5b6d83f..858414f7f 100644 --- a/pkgs/swarmauri/tests/unit/llms/FalAIImgGenModel_unit_test.py +++ b/pkgs/swarmauri/tests/unit/image_gens/FalAIImgGenModel_unit_test.py @@ -1,6 +1,6 @@ import pytest import os -from swarmauri.llms.concrete.FalAIImgGenModel import FalAIImgGenModel +from swarmauri.image_gens.concrete.FalAIImgGenModel import FalAIImgGenModel from dotenv import load_dotenv from swarmauri.utils.timeout_wrapper import timeout diff --git a/pkgs/swarmauri/tests/unit/llms/HyperbolicImgGenModel_unit_test.py b/pkgs/swarmauri/tests/unit/image_gens/HyperbolicImgGenModel_unit_test.py similarity index 97% rename from pkgs/swarmauri/tests/unit/llms/HyperbolicImgGenModel_unit_test.py rename to pkgs/swarmauri/tests/unit/image_gens/HyperbolicImgGenModel_unit_test.py index 1ad1b8d16..3772b4dce 100644 --- a/pkgs/swarmauri/tests/unit/llms/HyperbolicImgGenModel_unit_test.py +++ b/pkgs/swarmauri/tests/unit/image_gens/HyperbolicImgGenModel_unit_test.py @@ -1,6 +1,6 @@ import pytest import os -from swarmauri.llms.concrete.HyperbolicImgGenModel import HyperbolicImgGenModel +from swarmauri.image_gens.concrete.HyperbolicImgGenModel import HyperbolicImgGenModel from dotenv import load_dotenv from swarmauri.utils.timeout_wrapper import timeout diff --git a/pkgs/swarmauri/tests/unit/llms/OpenAIImgGenModel_unit_tesst.py b/pkgs/swarmauri/tests/unit/image_gens/OpenAIImgGenModel_unit_tesst.py similarity index 97% rename from pkgs/swarmauri/tests/unit/llms/OpenAIImgGenModel_unit_tesst.py rename to pkgs/swarmauri/tests/unit/image_gens/OpenAIImgGenModel_unit_tesst.py index 7780ba042..b22b9e6ea 100644 --- a/pkgs/swarmauri/tests/unit/llms/OpenAIImgGenModel_unit_tesst.py +++ b/pkgs/swarmauri/tests/unit/image_gens/OpenAIImgGenModel_unit_tesst.py @@ -1,7 +1,7 @@ import pytest import os from dotenv import load_dotenv -from swarmauri.llms.concrete.OpenAIImgGenModel import OpenAIImgGenModel +from swarmauri.image_gens.concrete.OpenAIImgGenModel import OpenAIImgGenModel from swarmauri.utils.timeout_wrapper import timeout load_dotenv() diff --git a/pkgs/swarmauri/tests/unit/llms/BlackForestImgGenModel_unit_test.py b/pkgs/swarmauri/tests/unit/llms/BlackForestImgGenModel_unit_test.py deleted file mode 100644 index 706d03b61..000000000 --- a/pkgs/swarmauri/tests/unit/llms/BlackForestImgGenModel_unit_test.py +++ /dev/null @@ -1,120 +0,0 @@ -import pytest -import os -from dotenv import load_dotenv -from swarmauri.llms.concrete.BlackForestImgGenModel import ( - BlackForestImgGenModel, -) - -from swarmauri.utils.timeout_wrapper import timeout - -load_dotenv() - -API_KEY = os.getenv("BLACKFOREST_API_KEY") - - -@pytest.fixture(scope="module") -def blackforest_imggen_model(): - if not API_KEY: - pytest.skip("Skipping due to environment variable not set") - model = BlackForestImgGenModel(api_key=API_KEY) - return model - - -def get_allowed_models(): - if not API_KEY: - return [] - model = BlackForestImgGenModel(api_key=API_KEY) - return model.allowed_models - - -@timeout(5) -@pytest.mark.unit -def test_model_resource(blackforest_imggen_model): - assert blackforest_imggen_model.resource == "LLM" - - -@timeout(5) -@pytest.mark.unit -def test_model_type(blackforest_imggen_model): - assert blackforest_imggen_model.type == "BlackForestImgGenModel" - - -@timeout(5) -@pytest.mark.unit -def test_serialization(blackforest_imggen_model): - assert ( - blackforest_imggen_model.id - == BlackForestImgGenModel.model_validate_json( - blackforest_imggen_model.model_dump_json() - ).id - ) - - -@timeout(5) -@pytest.mark.unit -def test_default_model_name(blackforest_imggen_model): - assert blackforest_imggen_model.name == "flux-pro" - - -@timeout(5) -@pytest.mark.parametrize("model_name", get_allowed_models()) -@pytest.mark.unit -def test_generate_image(blackforest_imggen_model, model_name): - model = blackforest_imggen_model - model.name = model_name - - prompt = "A cute dog playing in a park" - image_url = model.generate_image(prompt=prompt) - - assert isinstance(image_url, str) - assert image_url.startswith("http") - - -@timeout(5) -@pytest.mark.asyncio -@pytest.mark.parametrize("model_name", get_allowed_models()) -@pytest.mark.unit -async def test_agenerate_image(blackforest_imggen_model, model_name): - model = blackforest_imggen_model - model.name = model_name - - prompt = "A mountain with snow and a river" - image_url = await model.agenerate_image(prompt=prompt) - - assert isinstance(image_url, str) - assert image_url.startswith("http") - - -@timeout(5) -@pytest.mark.unit -def test_batch_generate(blackforest_imggen_model): - prompts = [ - "A futuristic city skyline", - "A tropical beach at sunset", - "A cup of coffee on a desk", - ] - - image_urls = blackforest_imggen_model.batch_generate(prompts=prompts) - - assert len(image_urls) == len(prompts) - for url in image_urls: - assert isinstance(url, str) - assert url.startswith("http") - - -@timeout(5) -@pytest.mark.asyncio -@pytest.mark.unit -async def test_abatch_generate(blackforest_imggen_model): - prompts = [ - "A space station in orbit", - "A lion resting in the savannah", - "A rainy day in a city", - ] - - image_urls = await blackforest_imggen_model.abatch_generate(prompts=prompts) - - assert len(image_urls) == len(prompts) - for url in image_urls: - assert isinstance(url, str) - assert url.startswith("http") From 6eabe4f43e286200ece84b1a9292df97cdc9f4e0 Mon Sep 17 00:00:00 2001 From: michaeldecent2 <111002205+MichaelDecent@users.noreply.github.com> Date: Thu, 21 Nov 2024 05:42:31 +0100 Subject: [PATCH 016/168] swarm - minor fix --- .../swarmauri/image_gens/concrete/BlackForestImgGenModel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgs/swarmauri/swarmauri/image_gens/concrete/BlackForestImgGenModel.py b/pkgs/swarmauri/swarmauri/image_gens/concrete/BlackForestImgGenModel.py index 202d5c631..d783c4204 100644 --- a/pkgs/swarmauri/swarmauri/image_gens/concrete/BlackForestImgGenModel.py +++ b/pkgs/swarmauri/swarmauri/image_gens/concrete/BlackForestImgGenModel.py @@ -15,8 +15,9 @@ class BlackForestImgGenModel(ImageGenBase): """ _BASE_URL: str = PrivateAttr("https://api.bfl.ml") - _client: httpx.Client = PrivateAttr() + _client: httpx.Client = PrivateAttr(default=None) _async_client: httpx.AsyncClient = PrivateAttr(default=None) + _headers: Dict[str, str] = PrivateAttr(default=None) api_key: str allowed_models: List[str] = ["flux-pro-1.1", "flux-pro", "flux-dev"] From 2f85274e0d6bfbe5feefee4abc0b46b9a6a86613 Mon Sep 17 00:00:00 2001 From: michaeldecent2 <111002205+MichaelDecent@users.noreply.github.com> Date: Thu, 21 Nov 2024 11:59:46 +0100 Subject: [PATCH 017/168] comm - Refactor imports in vector store unit tests and add lazy loading utility --- .../document_stores/concrete/__init__.py | 13 +- .../embeddings/concrete/__init__.py | 12 + .../llms/concrete/__init__.py | 14 +- .../measurements/concrete/__init__.py | 17 +- .../parsers/concrete/__init__.py | 19 +- .../retrievers/concrete/__init__.py | 13 +- .../toolkits/concrete/__init__.py | 11 +- .../tools/concrete/__init__.py | 62 +-- .../vector_stores/base/__init__.py | 0 .../{ => concrete}/AnnoyVectorStore.py | 0 .../{ => concrete}/CloudQdrantVectorStore.py | 0 .../CloudWeaviateVectorStore.py | 434 +++++++++--------- .../{ => concrete}/Doc2VecVectorStore.py | 0 .../{ => concrete}/DuckDBVectorStore.py | 0 .../{ => concrete}/MlmVectorStore.py | 0 .../{ => concrete}/Neo4jVectorStore.py | 0 .../PersistentChromaDBVectorStore.py | 0 .../PersistentQdrantVectorStore.py | 0 .../{ => concrete}/PineconeVectorStore.py | 0 .../{ => concrete}/RedisVectorStore.py | 0 .../vector_stores/concrete/__init__.py | 20 + .../vector_stores/AnnoyVectorStore_test.py | 2 +- .../CloudQdrantVectorStore_test.py | 2 +- .../CloudWeaviateVectorStore_test.py | 2 +- .../Doc2VecVectorStore_unit_test.py | 2 +- .../DuckDBVectorStore_unit_test.py | 2 +- .../vector_stores/MlmVectorStore_unit_test.py | 2 +- .../vector_stores/Neo4jVectorStore_test.py | 2 +- .../PersistentChromadbVectorStore_test.py | 2 +- .../PersistentQdrantVectorStore_test.py | 2 +- .../vector_stores/PineconeVectorStore_test.py | 2 +- .../vector_stores/RedisVectorStore_test.py | 4 +- .../swarmauri/swarmauri/utils/_lazy_import.py | 22 + 33 files changed, 381 insertions(+), 280 deletions(-) create mode 100644 pkgs/community/swarmauri_community/vector_stores/base/__init__.py rename pkgs/community/swarmauri_community/vector_stores/{ => concrete}/AnnoyVectorStore.py (100%) rename pkgs/community/swarmauri_community/vector_stores/{ => concrete}/CloudQdrantVectorStore.py (100%) rename pkgs/community/swarmauri_community/vector_stores/{ => concrete}/CloudWeaviateVectorStore.py (97%) rename pkgs/community/swarmauri_community/vector_stores/{ => concrete}/Doc2VecVectorStore.py (100%) rename pkgs/community/swarmauri_community/vector_stores/{ => concrete}/DuckDBVectorStore.py (100%) rename pkgs/community/swarmauri_community/vector_stores/{ => concrete}/MlmVectorStore.py (100%) rename pkgs/community/swarmauri_community/vector_stores/{ => concrete}/Neo4jVectorStore.py (100%) rename pkgs/community/swarmauri_community/vector_stores/{ => concrete}/PersistentChromaDBVectorStore.py (100%) rename pkgs/community/swarmauri_community/vector_stores/{ => concrete}/PersistentQdrantVectorStore.py (100%) rename pkgs/community/swarmauri_community/vector_stores/{ => concrete}/PineconeVectorStore.py (100%) rename pkgs/community/swarmauri_community/vector_stores/{ => concrete}/RedisVectorStore.py (100%) create mode 100644 pkgs/community/swarmauri_community/vector_stores/concrete/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/utils/_lazy_import.py diff --git a/pkgs/community/swarmauri_community/document_stores/concrete/__init__.py b/pkgs/community/swarmauri_community/document_stores/concrete/__init__.py index a44ceb5c2..941fc5618 100644 --- a/pkgs/community/swarmauri_community/document_stores/concrete/__init__.py +++ b/pkgs/community/swarmauri_community/document_stores/concrete/__init__.py @@ -1,3 +1,10 @@ -from swarmauri_community.document_stores.concrete.RedisDocumentStore import ( - RedisDocumentStore, -) +from swarmauri.utils._lazy_import import _lazy_import + +documents_stores_files = [ + ("swarmauri_community.documents_stores.concrete.RedisDocumentStore", "RedisDocumentStore"), +] + +for module_name, class_name in documents_stores_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +__all__ = [class_name for _, class_name in documents_stores_files] diff --git a/pkgs/community/swarmauri_community/embeddings/concrete/__init__.py b/pkgs/community/swarmauri_community/embeddings/concrete/__init__.py index e69de29bb..7bcc482d7 100644 --- a/pkgs/community/swarmauri_community/embeddings/concrete/__init__.py +++ b/pkgs/community/swarmauri_community/embeddings/concrete/__init__.py @@ -0,0 +1,12 @@ +from swarmauri.utils._lazy_import import _lazy_import + + +embeddings_files = [ + ("swarmauri_community.embeddings.concrete.Doc2VecEmbedding", "Doc2VecEmbedding"), + ("swarmauri_community.embeddings.concrete.MlmEmbedding", "MlmEmbedding"), +] + +for module_name, class_name in embeddings_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +__all__ = [class_name for _, class_name in embeddings_files] diff --git a/pkgs/community/swarmauri_community/llms/concrete/__init__.py b/pkgs/community/swarmauri_community/llms/concrete/__init__.py index a8fa703c0..5c2266ce5 100644 --- a/pkgs/community/swarmauri_community/llms/concrete/__init__.py +++ b/pkgs/community/swarmauri_community/llms/concrete/__init__.py @@ -1,4 +1,12 @@ -from swarmauri_community.llms.concrete.LeptonAIImgGenModel import LeptonAIImgGenModel -from swarmauri_community.llms.concrete.LeptonAIModel import LeptonAIModel +from swarmauri.utils._lazy_import import _lazy_import -__all__ = ["LeptonAIImgGenModel", "LeptonAIModel"] +llms_files = [ + ("swarmauri_community.llms.concrete.LeptonAIImgGenModel", "LeptonAIImgGenModel"), + ("swarmauri_community.llms.concrete.LeptonAIModel", "LeptonAIModel"), + ("swarmauri_community.llms.concrete.PytesseractImg2TextModel", "PytesseractImg2TextModel"), +] + +for module_name, class_name in llms_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +__all__ = [class_name for _, class_name in llms_files] diff --git a/pkgs/community/swarmauri_community/measurements/concrete/__init__.py b/pkgs/community/swarmauri_community/measurements/concrete/__init__.py index 276716315..a748c6a0e 100644 --- a/pkgs/community/swarmauri_community/measurements/concrete/__init__.py +++ b/pkgs/community/swarmauri_community/measurements/concrete/__init__.py @@ -1,6 +1,11 @@ -from swarmauri_community.measurements.concrete.MutualInformationMeasurement import ( - MutualInformationMeasurement, -) -from swarmauri_community.measurements.concrete.TokenCountEstimatorMeasurement import ( - TokenCountEstimatorMeasurement, -) +from swarmauri.utils._lazy_import import _lazy_import + +measurement_files = [ + ("swarmauri_community.measurements.concrete.MutualInformationMeasurement", "MutualInformationMeasurement"), + ("swarmauri_community.measurements.concrete.TokenCountEstimatorMeasurement", "TokenCountEstimatorMeasurement"), +] + +for module_name, class_name in measurement_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +__all__ = [class_name for _, class_name in measurement_files] diff --git a/pkgs/community/swarmauri_community/parsers/concrete/__init__.py b/pkgs/community/swarmauri_community/parsers/concrete/__init__.py index b5d547c4e..3808b4733 100644 --- a/pkgs/community/swarmauri_community/parsers/concrete/__init__.py +++ b/pkgs/community/swarmauri_community/parsers/concrete/__init__.py @@ -1,3 +1,16 @@ -from swarmauri_community.parsers.concrete.FitzPdfParser import PDFtoTextParser -from swarmauri_community.parsers.concrete.PyPDF2Parser import PyPDF2Parser -from swarmauri_community.parsers.concrete.PyPDFTKParser import PyPDFTKParser +from swarmauri.utils._lazy_import import _lazy_import + +parsers_files = [ + ("swarmauri_community.parsers.concrete.BERTEmbeddingParser", "BERTEmbeddingParser"), + ("swarmauri_community.parsers.concrete.EntityRecognitionParser", "EntityRecognitionParser"), + ("swarmauri_community.parsers.concrete.FitzPdfParser", "FitzPdfParser"), + ("swarmauri_community.parsers.concrete.PyPDF2Parser", "PyPDF2Parser"), + ("swarmauri_community.parsers.concrete.PyPDFTKParser", "PyPDFTKParser"), + ("swarmauri_community.parsers.concrete.TextBlobNounParser", "TextBlobNounParser"), + ("swarmauri_community.parsers.concrete.TextBlobSentenceParser", "TextBlobSentenceParser"), +] + +for module_name, class_name in parsers_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +__all__ = [class_name for _, class_name in parsers_files] diff --git a/pkgs/community/swarmauri_community/retrievers/concrete/__init__.py b/pkgs/community/swarmauri_community/retrievers/concrete/__init__.py index 000e57ffe..ec089ab66 100644 --- a/pkgs/community/swarmauri_community/retrievers/concrete/__init__.py +++ b/pkgs/community/swarmauri_community/retrievers/concrete/__init__.py @@ -1,5 +1,10 @@ -# -*- coding: utf-8 -*- +from swarmauri.utils._lazy_import import _lazy_import -from swarmauri_community.retrievers.concrete.RedisDocumentRetriever import ( - RedisDocumentRetriever, -) +retriever_files = [ + ("swarmauri_community.retrievers.concrete.RedisDocumentRetriever", "RedisDocumentRetriever"), +] + +for module_name, class_name in retriever_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +__all__ = [class_name for _, class_name in retriever_files] diff --git a/pkgs/community/swarmauri_community/toolkits/concrete/__init__.py b/pkgs/community/swarmauri_community/toolkits/concrete/__init__.py index 129ad1dc9..6aca27ddc 100644 --- a/pkgs/community/swarmauri_community/toolkits/concrete/__init__.py +++ b/pkgs/community/swarmauri_community/toolkits/concrete/__init__.py @@ -1 +1,10 @@ -from swarmauri_community.toolkits.concrete.GithubToolkit import * +from swarmauri.utils._lazy_import import _lazy_import + +toolkits_files = [ + ("swarmauri_community.toolkits.concrete.GithubToolkit", "GithubToolkit"), +] + +for module_name, class_name in toolkits_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +__all__ = [class_name for _, class_name in toolkits_files] diff --git a/pkgs/community/swarmauri_community/tools/concrete/__init__.py b/pkgs/community/swarmauri_community/tools/concrete/__init__.py index 9db51c17f..fe50a5544 100644 --- a/pkgs/community/swarmauri_community/tools/concrete/__init__.py +++ b/pkgs/community/swarmauri_community/tools/concrete/__init__.py @@ -1,31 +1,33 @@ -from swarmauri_community.tools.concrete.CaptchaGeneratorTool import CaptchaGeneratorTool -from swarmauri_community.tools.concrete.DaleChallReadabilityTool import ( - DaleChallReadabilityTool, -) -from swarmauri_community.tools.concrete.DownloadPdfTool import DownloadPDFTool -from swarmauri_community.tools.concrete.EntityRecognitionTool import ( - EntityRecognitionTool, -) -from swarmauri_community.tools.concrete.FoliumTool import FoliumTool -from swarmauri_community.tools.concrete.GithubBranchTool import GithubBranchTool -from swarmauri_community.tools.concrete.GithubCommitTool import GithubCommitTool -from swarmauri_community.tools.concrete.GithubIssueTool import GithubIssueTool -from swarmauri_community.tools.concrete.GithubPRTool import GithubPRTool -from swarmauri_community.tools.concrete.GithubRepoTool import GithubRepoTool -from swarmauri_community.tools.concrete.GithubTool import GithubTool -from swarmauri_community.tools.concrete.GmailReadTool import GmailReadTool -from swarmauri_community.tools.concrete.GmailSendTool import GmailSendTool -from swarmauri_community.tools.concrete.LexicalDensityTool import LexicalDensityTool -from swarmauri_community.tools.concrete.PsutilTool import PsutilTool -from swarmauri_community.tools.concrete.QrCodeGeneratorTool import QrCodeGeneratorTool -from swarmauri_community.tools.concrete.SentenceComplexityTool import ( - SentenceComplexityTool, -) -from swarmauri_community.tools.concrete.SentimentAnalysisTool import ( - SentimentAnalysisTool, -) -from swarmauri_community.tools.concrete.SMOGIndexTool import SMOGIndexTool -from swarmauri_community.tools.concrete.WebScrapingTool import WebScrapingTool -from swarmauri_community.tools.concrete.ZapierHookTool import ZapierHookTool +from swarmauri.utils._lazy_import import _lazy_import -# from swarmauri_community.tools.concrete.PaCMAPTool import PaCMAPTool +tool_files = [ + ("swarmauri_community.tools.concrete.CaptchaGeneratorTool", "CaptchaGeneratorTool"), + ("swarmauri_community.tools.concrete.DaleChallReadabilityTool", "DaleChallReadabilityTool"), + ("swarmauri_community.tools.concrete.DownloadPdfTool", "DownloadPDFTool"), + ("swarmauri_community.tools.concrete.EntityRecognitionTool", "EntityRecognitionTool"), + ("swarmauri_community.tools.concrete.FoliumTool", "FoliumTool"), + ("swarmauri_community.tools.concrete.GithubBranchTool", "GithubBranchTool"), + ("swarmauri_community.tools.concrete.GithubCommitTool", "GithubCommitTool"), + ("swarmauri_community.tools.concrete.GithubIssueTool", "GithubIssueTool"), + ("swarmauri_community.tools.concrete.GithubPRTool", "GithubPRTool"), + ("swarmauri_community.tools.concrete.GithubRepoTool", "GithubRepoTool"), + ("swarmauri_community.tools.concrete.GithubTool", "GithubTool"), + ("swarmauri_community.tools.concrete.GmailReadTool", "GmailReadTool"), + ("swarmauri_community.tools.concrete.GmailSendTool", "GmailSendTool"), + ("swarmauri_community.tools.concrete.LexicalDensityTool", "LexicalDensityTool"), + ("swarmauri_community.tools.concrete.PsutilTool", "PsutilTool"), + ("swarmauri_community.tools.concrete.QrCodeGeneratorTool", "QrCodeGeneratorTool"), + ("swarmauri_community.tools.concrete.SentenceComplexityTool", "SentenceComplexityTool"), + ("swarmauri_community.tools.concrete.SentimentAnalysisTool", "SentimentAnalysisTool"), + ("swarmauri_community.tools.concrete.SMOGIndexTool", "SMOGIndexTool"), + ("swarmauri_community.tools.concrete.WebScrapingTool", "WebScrapingTool"), + ("swarmauri_community.tools.concrete.ZapierHookTool", "ZapierHookTool"), + # ("swarmauri_community.tools.concrete.PaCMAPTool", "PaCMAPTool"), +] + +# Lazy loading of tools, storing them in variables +for module_name, class_name in tool_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded tools to __all__ +__all__ = [class_name for _, class_name in tool_files] diff --git a/pkgs/community/swarmauri_community/vector_stores/base/__init__.py b/pkgs/community/swarmauri_community/vector_stores/base/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/community/swarmauri_community/vector_stores/AnnoyVectorStore.py b/pkgs/community/swarmauri_community/vector_stores/concrete/AnnoyVectorStore.py similarity index 100% rename from pkgs/community/swarmauri_community/vector_stores/AnnoyVectorStore.py rename to pkgs/community/swarmauri_community/vector_stores/concrete/AnnoyVectorStore.py diff --git a/pkgs/community/swarmauri_community/vector_stores/CloudQdrantVectorStore.py b/pkgs/community/swarmauri_community/vector_stores/concrete/CloudQdrantVectorStore.py similarity index 100% rename from pkgs/community/swarmauri_community/vector_stores/CloudQdrantVectorStore.py rename to pkgs/community/swarmauri_community/vector_stores/concrete/CloudQdrantVectorStore.py diff --git a/pkgs/community/swarmauri_community/vector_stores/CloudWeaviateVectorStore.py b/pkgs/community/swarmauri_community/vector_stores/concrete/CloudWeaviateVectorStore.py similarity index 97% rename from pkgs/community/swarmauri_community/vector_stores/CloudWeaviateVectorStore.py rename to pkgs/community/swarmauri_community/vector_stores/concrete/CloudWeaviateVectorStore.py index 9e55fc0ca..0e1ffced7 100644 --- a/pkgs/community/swarmauri_community/vector_stores/CloudWeaviateVectorStore.py +++ b/pkgs/community/swarmauri_community/vector_stores/concrete/CloudWeaviateVectorStore.py @@ -1,218 +1,218 @@ -from typing import List, Union, Literal, Optional -from pydantic import BaseModel, PrivateAttr -import uuid as ud -import weaviate -from weaviate.classes.init import Auth -from weaviate.util import generate_uuid5 -from weaviate.classes.query import MetadataQuery - -from swarmauri.documents.concrete.Document import Document -from swarmauri.embeddings.concrete.Doc2VecEmbedding import Doc2VecEmbedding -from swarmauri.vectors.concrete.Vector import Vector - -from swarmauri.vector_stores.base.VectorStoreBase import VectorStoreBase -from swarmauri.vector_stores.base.VectorStoreRetrieveMixin import VectorStoreRetrieveMixin -from swarmauri.vector_stores.base.VectorStoreSaveLoadMixin import VectorStoreSaveLoadMixin -from swarmauri.vector_stores.base.VectorStoreCloudMixin import VectorStoreCloudMixin - - -class CloudWeaviateVectorStore(VectorStoreSaveLoadMixin, VectorStoreRetrieveMixin, VectorStoreBase, VectorStoreCloudMixin): - type: Literal["CloudWeaviateVectorStore"] = "CloudWeaviateVectorStore" - - - # Private attributes - _client: Optional[weaviate.Client] = PrivateAttr(default=None) - _embedder: Doc2VecEmbedding = PrivateAttr(default=None) - _namespace_uuid: ud.UUID = PrivateAttr(default_factory=ud.uuid4) - - def __init__(self, **data): - super().__init__(**data) - - # Initialize the vectorizer and Weaviate client - self._embedder = Doc2VecEmbedding(vector_size=self.vector_size) - # self._initialize_client() - - def connect(self, **kwargs): - """ - Initialize the Weaviate client. - """ - if self._client is None: - self._client = weaviate.connect_to_weaviate_cloud( - cluster_url=self.url, - auth_credentials=Auth.api_key(self.api_key), - headers=kwargs.get("headers", {}) - ) - - def disconnect(self) -> None: - """ - Disconnects from the Qdrant cloud vector store. - """ - if self.client is not None: - self.client = None - - def add_document(self, document: Document) -> None: - """ - Add a single document to the vector store. - - :param document: Document to add - """ - try: - collection = self._client.collections.get(self.collection_name) - - # Generate or use existing embedding - embedding = document.embedding or self._embedder.fit_transform([document.content])[0] - - data_object = { - "content": document.content, - "metadata": document.metadata, - } - - # Generate UUID for document - uuid = ( - str(ud.uuid5(self._namespace_uuid, document.id)) - if document.id - else generate_uuid5(data_object) - ) - - collection.data.insert( - properties=data_object, - vector=embedding.value, - uuid=uuid, - ) - - print(f"Document '{document.id}' added to Weaviate.") - except Exception as e: - print(f"Error adding document '{document.id}': {e}") - raise - - def add_documents(self, documents: List[Document]) -> None: - """ - Add multiple documents to the vector store. - - :param documents: List of documents to add - """ - try: - for document in documents: - self.add_document(document) - - print(f"{len(documents)} documents added to Weaviate.") - except Exception as e: - print(f"Error adding documents: {e}") - raise - - def get_document(self, id: str) -> Union[Document, None]: - """ - Retrieve a document by its ID. - - :param id: Document ID - :return: Document object or None if not found - """ - try: - collection = self._client.collections.get(self.collection_name) - - result = collection.query.fetch_object_by_id(ud.uuid5(self._namespace_uuid, id)) - - if result: - return Document( - id=id, - content=result.properties["content"], - metadata=result.properties["metadata"], - ) - return None - except Exception as e: - print(f"Error retrieving document '{id}': {e}") - return None - - def get_all_documents(self) -> List[Document]: - """ - Retrieve all documents from the vector store. - - :return: List of Document objects - """ - try: - collection = self._client.collections.get(self.collection_name) - # return collection - documents = [ - Document( - content=item.properties["content"], - metadata=item.properties["metadata"], - embedding=Vector(value=list(item.vector.values())[0]), - ) - for item in collection.iterator(include_vector=True) - ] - return documents - except Exception as e: - print(f"Error retrieving all documents: {e}") - return [] - - def delete_document(self, id: str) -> None: - """ - Delete a document by its ID. - - :param id: Document ID - """ - try: - collection = self._client.collections.get(self.collection_name) - collection.data.delete_by_id(ud.uuid5(self._namespace_uuid, id)) - print(f"Document '{id}' has been deleted from Weaviate.") - except Exception as e: - print(f"Error deleting document '{id}': {e}") - raise - - def update_document(self, id: str, document: Document) -> None: - """ - Update an existing document. - - :param id: Document ID - :param updated_document: Document object with updated data - """ - self.delete_document(id) - self.add_document(document) - - def retrieve(self, query: str, top_k: int = 5) -> List[Document]: - """ - Retrieve the top_k most relevant documents based on the given query. - - :param query: Query string - :param top_k: Number of top similar documents to retrieve - :return: List of Document objects - """ - try: - collection = self._client.collections.get(self.collection_name) - query_vector = self._embedder.infer_vector(query) - response = collection.query.near_vector( - near_vector=query_vector.value, - limit=top_k, - return_metadata=MetadataQuery(distance=True), - ) - - documents = [ - Document( - # id=res.id, - content=res.properties["content"], - metadata=res.properties["metadata"], - ) - for res in response.objects - ] - return documents - except Exception as e: - print(f"Error retrieving documents for query '{query}': {e}") - return [] - - def close(self): - """ - Close the connection to the Weaviate server. - """ - if self._client: - self._client.close() - - def model_dump_json(self, *args, **kwargs) -> str: - # Call the disconnect method before serialization - self.disconnect() - - # Now proceed with the usual JSON serialization - return super().model_dump_json(*args, **kwargs) - - - def __del__(self): +from typing import List, Union, Literal, Optional +from pydantic import BaseModel, PrivateAttr +import uuid as ud +import weaviate +from weaviate.classes.init import Auth +from weaviate.util import generate_uuid5 +from weaviate.classes.query import MetadataQuery + +from swarmauri.documents.concrete.Document import Document +from swarmauri.embeddings.concrete.Doc2VecEmbedding import Doc2VecEmbedding +from swarmauri.vectors.concrete.Vector import Vector + +from swarmauri.vector_stores.base.VectorStoreBase import VectorStoreBase +from swarmauri.vector_stores.base.VectorStoreRetrieveMixin import VectorStoreRetrieveMixin +from swarmauri.vector_stores.base.VectorStoreSaveLoadMixin import VectorStoreSaveLoadMixin +from swarmauri.vector_stores.base.VectorStoreCloudMixin import VectorStoreCloudMixin + + +class CloudWeaviateVectorStore(VectorStoreSaveLoadMixin, VectorStoreRetrieveMixin, VectorStoreBase, VectorStoreCloudMixin): + type: Literal["CloudWeaviateVectorStore"] = "CloudWeaviateVectorStore" + + + # Private attributes + _client: Optional[weaviate.Client] = PrivateAttr(default=None) + _embedder: Doc2VecEmbedding = PrivateAttr(default=None) + _namespace_uuid: ud.UUID = PrivateAttr(default_factory=ud.uuid4) + + def __init__(self, **data): + super().__init__(**data) + + # Initialize the vectorizer and Weaviate client + self._embedder = Doc2VecEmbedding(vector_size=self.vector_size) + # self._initialize_client() + + def connect(self, **kwargs): + """ + Initialize the Weaviate client. + """ + if self._client is None: + self._client = weaviate.connect_to_weaviate_cloud( + cluster_url=self.url, + auth_credentials=Auth.api_key(self.api_key), + headers=kwargs.get("headers", {}) + ) + + def disconnect(self) -> None: + """ + Disconnects from the Qdrant cloud vector store. + """ + if self.client is not None: + self.client = None + + def add_document(self, document: Document) -> None: + """ + Add a single document to the vector store. + + :param document: Document to add + """ + try: + collection = self._client.collections.get(self.collection_name) + + # Generate or use existing embedding + embedding = document.embedding or self._embedder.fit_transform([document.content])[0] + + data_object = { + "content": document.content, + "metadata": document.metadata, + } + + # Generate UUID for document + uuid = ( + str(ud.uuid5(self._namespace_uuid, document.id)) + if document.id + else generate_uuid5(data_object) + ) + + collection.data.insert( + properties=data_object, + vector=embedding.value, + uuid=uuid, + ) + + print(f"Document '{document.id}' added to Weaviate.") + except Exception as e: + print(f"Error adding document '{document.id}': {e}") + raise + + def add_documents(self, documents: List[Document]) -> None: + """ + Add multiple documents to the vector store. + + :param documents: List of documents to add + """ + try: + for document in documents: + self.add_document(document) + + print(f"{len(documents)} documents added to Weaviate.") + except Exception as e: + print(f"Error adding documents: {e}") + raise + + def get_document(self, id: str) -> Union[Document, None]: + """ + Retrieve a document by its ID. + + :param id: Document ID + :return: Document object or None if not found + """ + try: + collection = self._client.collections.get(self.collection_name) + + result = collection.query.fetch_object_by_id(ud.uuid5(self._namespace_uuid, id)) + + if result: + return Document( + id=id, + content=result.properties["content"], + metadata=result.properties["metadata"], + ) + return None + except Exception as e: + print(f"Error retrieving document '{id}': {e}") + return None + + def get_all_documents(self) -> List[Document]: + """ + Retrieve all documents from the vector store. + + :return: List of Document objects + """ + try: + collection = self._client.collections.get(self.collection_name) + # return collection + documents = [ + Document( + content=item.properties["content"], + metadata=item.properties["metadata"], + embedding=Vector(value=list(item.vector.values())[0]), + ) + for item in collection.iterator(include_vector=True) + ] + return documents + except Exception as e: + print(f"Error retrieving all documents: {e}") + return [] + + def delete_document(self, id: str) -> None: + """ + Delete a document by its ID. + + :param id: Document ID + """ + try: + collection = self._client.collections.get(self.collection_name) + collection.data.delete_by_id(ud.uuid5(self._namespace_uuid, id)) + print(f"Document '{id}' has been deleted from Weaviate.") + except Exception as e: + print(f"Error deleting document '{id}': {e}") + raise + + def update_document(self, id: str, document: Document) -> None: + """ + Update an existing document. + + :param id: Document ID + :param updated_document: Document object with updated data + """ + self.delete_document(id) + self.add_document(document) + + def retrieve(self, query: str, top_k: int = 5) -> List[Document]: + """ + Retrieve the top_k most relevant documents based on the given query. + + :param query: Query string + :param top_k: Number of top similar documents to retrieve + :return: List of Document objects + """ + try: + collection = self._client.collections.get(self.collection_name) + query_vector = self._embedder.infer_vector(query) + response = collection.query.near_vector( + near_vector=query_vector.value, + limit=top_k, + return_metadata=MetadataQuery(distance=True), + ) + + documents = [ + Document( + # id=res.id, + content=res.properties["content"], + metadata=res.properties["metadata"], + ) + for res in response.objects + ] + return documents + except Exception as e: + print(f"Error retrieving documents for query '{query}': {e}") + return [] + + def close(self): + """ + Close the connection to the Weaviate server. + """ + if self._client: + self._client.close() + + def model_dump_json(self, *args, **kwargs) -> str: + # Call the disconnect method before serialization + self.disconnect() + + # Now proceed with the usual JSON serialization + return super().model_dump_json(*args, **kwargs) + + + def __del__(self): self.close() \ No newline at end of file diff --git a/pkgs/community/swarmauri_community/vector_stores/Doc2VecVectorStore.py b/pkgs/community/swarmauri_community/vector_stores/concrete/Doc2VecVectorStore.py similarity index 100% rename from pkgs/community/swarmauri_community/vector_stores/Doc2VecVectorStore.py rename to pkgs/community/swarmauri_community/vector_stores/concrete/Doc2VecVectorStore.py diff --git a/pkgs/community/swarmauri_community/vector_stores/DuckDBVectorStore.py b/pkgs/community/swarmauri_community/vector_stores/concrete/DuckDBVectorStore.py similarity index 100% rename from pkgs/community/swarmauri_community/vector_stores/DuckDBVectorStore.py rename to pkgs/community/swarmauri_community/vector_stores/concrete/DuckDBVectorStore.py diff --git a/pkgs/community/swarmauri_community/vector_stores/MlmVectorStore.py b/pkgs/community/swarmauri_community/vector_stores/concrete/MlmVectorStore.py similarity index 100% rename from pkgs/community/swarmauri_community/vector_stores/MlmVectorStore.py rename to pkgs/community/swarmauri_community/vector_stores/concrete/MlmVectorStore.py diff --git a/pkgs/community/swarmauri_community/vector_stores/Neo4jVectorStore.py b/pkgs/community/swarmauri_community/vector_stores/concrete/Neo4jVectorStore.py similarity index 100% rename from pkgs/community/swarmauri_community/vector_stores/Neo4jVectorStore.py rename to pkgs/community/swarmauri_community/vector_stores/concrete/Neo4jVectorStore.py diff --git a/pkgs/community/swarmauri_community/vector_stores/PersistentChromaDBVectorStore.py b/pkgs/community/swarmauri_community/vector_stores/concrete/PersistentChromaDBVectorStore.py similarity index 100% rename from pkgs/community/swarmauri_community/vector_stores/PersistentChromaDBVectorStore.py rename to pkgs/community/swarmauri_community/vector_stores/concrete/PersistentChromaDBVectorStore.py diff --git a/pkgs/community/swarmauri_community/vector_stores/PersistentQdrantVectorStore.py b/pkgs/community/swarmauri_community/vector_stores/concrete/PersistentQdrantVectorStore.py similarity index 100% rename from pkgs/community/swarmauri_community/vector_stores/PersistentQdrantVectorStore.py rename to pkgs/community/swarmauri_community/vector_stores/concrete/PersistentQdrantVectorStore.py diff --git a/pkgs/community/swarmauri_community/vector_stores/PineconeVectorStore.py b/pkgs/community/swarmauri_community/vector_stores/concrete/PineconeVectorStore.py similarity index 100% rename from pkgs/community/swarmauri_community/vector_stores/PineconeVectorStore.py rename to pkgs/community/swarmauri_community/vector_stores/concrete/PineconeVectorStore.py diff --git a/pkgs/community/swarmauri_community/vector_stores/RedisVectorStore.py b/pkgs/community/swarmauri_community/vector_stores/concrete/RedisVectorStore.py similarity index 100% rename from pkgs/community/swarmauri_community/vector_stores/RedisVectorStore.py rename to pkgs/community/swarmauri_community/vector_stores/concrete/RedisVectorStore.py diff --git a/pkgs/community/swarmauri_community/vector_stores/concrete/__init__.py b/pkgs/community/swarmauri_community/vector_stores/concrete/__init__.py new file mode 100644 index 000000000..f920d22b4 --- /dev/null +++ b/pkgs/community/swarmauri_community/vector_stores/concrete/__init__.py @@ -0,0 +1,20 @@ +from swarmauri.utils._lazy_import import _lazy_import + +vector_store_files = [ + ("swarmauri_community.vector_stores.concrete.AnnoyVectorStore", "AnnoyVectorStore"), + ("swarmauri_community.vector_stores.concrete.CloudQdrantVectorStore", "CloudQdrantVectorStore"), + ("swarmauri_community.vector_stores.concrete.CloudWeaviateVectorStore", "CloudWeaviateVectorStore"), + ("swarmauri_community.vector_stores.concrete.Doc2VecVectorStore", "Doc2VecVectorStore"), + ("swarmauri_community.vector_stores.concrete.DuckDBVectorStore", "DuckDBVectorStore"), + ("swarmauri_community.vector_stores.concrete.MlmVectorStore", "MlmVectorStore"), + ("swarmauri_community.vector_stores.concrete.Neo4jVectorStore", "Neo4jVectorStore"), + ("swarmauri_community.vector_stores.concrete.PersistentChromaDBVectorStore", "PersistentChromaDBVectorStore"), + ("swarmauri_community.vector_stores.concrete.PersistentQdrantVectorStore", "PersistentQdrantVectorStore"), + ("swarmauri_community.vector_stores.concrete.PineconeVectorStore", "PineconeVectorStore"), + ("swarmauri_community.vector_stores.concrete.RedisVectorStore", "RedisVectorStore"), +] + +for module_name, class_name in vector_store_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +__all__ = [class_name for _, class_name in vector_store_files] diff --git a/pkgs/community/tests/unit/vector_stores/AnnoyVectorStore_test.py b/pkgs/community/tests/unit/vector_stores/AnnoyVectorStore_test.py index 9f2ce8320..cee7afddd 100644 --- a/pkgs/community/tests/unit/vector_stores/AnnoyVectorStore_test.py +++ b/pkgs/community/tests/unit/vector_stores/AnnoyVectorStore_test.py @@ -1,6 +1,6 @@ import pytest from swarmauri.documents.concrete.Document import Document -from swarmauri_community.vector_stores.AnnoyVectorStore import AnnoyVectorStore +from swarmauri_community.vector_stores.concrete.AnnoyVectorStore import AnnoyVectorStore # Fixture for creating an AnnoyVectorStore instance diff --git a/pkgs/community/tests/unit/vector_stores/CloudQdrantVectorStore_test.py b/pkgs/community/tests/unit/vector_stores/CloudQdrantVectorStore_test.py index 2b9028c72..26ee25841 100644 --- a/pkgs/community/tests/unit/vector_stores/CloudQdrantVectorStore_test.py +++ b/pkgs/community/tests/unit/vector_stores/CloudQdrantVectorStore_test.py @@ -1,7 +1,7 @@ import os import pytest from swarmauri.documents.concrete.Document import Document -from swarmauri_community.vector_stores.CloudQdrantVectorStore import ( +from swarmauri_community.vector_stores.concrete.CloudQdrantVectorStore import ( CloudQdrantVectorStore, ) diff --git a/pkgs/community/tests/unit/vector_stores/CloudWeaviateVectorStore_test.py b/pkgs/community/tests/unit/vector_stores/CloudWeaviateVectorStore_test.py index 051d6aa6b..9bad53191 100644 --- a/pkgs/community/tests/unit/vector_stores/CloudWeaviateVectorStore_test.py +++ b/pkgs/community/tests/unit/vector_stores/CloudWeaviateVectorStore_test.py @@ -1,7 +1,7 @@ import os import pytest from swarmauri.documents.concrete.Document import Document -from swarmauri_community.vector_stores.CloudWeaviateVectorStore import ( +from swarmauri_community.vector_stores.concrete.CloudWeaviateVectorStore import ( CloudWeaviateVectorStore, ) from dotenv import load_dotenv diff --git a/pkgs/community/tests/unit/vector_stores/Doc2VecVectorStore_unit_test.py b/pkgs/community/tests/unit/vector_stores/Doc2VecVectorStore_unit_test.py index 01c8eb634..497e8a45f 100644 --- a/pkgs/community/tests/unit/vector_stores/Doc2VecVectorStore_unit_test.py +++ b/pkgs/community/tests/unit/vector_stores/Doc2VecVectorStore_unit_test.py @@ -1,6 +1,6 @@ import pytest from swarmauri.documents.concrete.Document import Document -from swarmauri_community.vector_stores.Doc2VecVectorStore import Doc2VecVectorStore +from swarmauri_community.vector_stores.concrete.Doc2VecVectorStore import Doc2VecVectorStore @pytest.mark.unit diff --git a/pkgs/community/tests/unit/vector_stores/DuckDBVectorStore_unit_test.py b/pkgs/community/tests/unit/vector_stores/DuckDBVectorStore_unit_test.py index 28bd33080..0b247ccd8 100644 --- a/pkgs/community/tests/unit/vector_stores/DuckDBVectorStore_unit_test.py +++ b/pkgs/community/tests/unit/vector_stores/DuckDBVectorStore_unit_test.py @@ -2,7 +2,7 @@ import os import json from swarmauri.documents.concrete.Document import Document -from swarmauri_community.vector_stores.DuckDBVectorStore import DuckDBVectorStore +from swarmauri_community.vector_stores.concrete.DuckDBVectorStore import DuckDBVectorStore @pytest.fixture(params=[":memory:", "test_db.db"]) diff --git a/pkgs/community/tests/unit/vector_stores/MlmVectorStore_unit_test.py b/pkgs/community/tests/unit/vector_stores/MlmVectorStore_unit_test.py index 5f486aebb..1c3dc9273 100644 --- a/pkgs/community/tests/unit/vector_stores/MlmVectorStore_unit_test.py +++ b/pkgs/community/tests/unit/vector_stores/MlmVectorStore_unit_test.py @@ -1,6 +1,6 @@ import pytest from swarmauri.documents.concrete.Document import Document -from swarmauri_community.vector_stores.MlmVectorStore import MlmVectorStore +from swarmauri_community.vector_stores.concrete.MlmVectorStore import MlmVectorStore @pytest.mark.unit diff --git a/pkgs/community/tests/unit/vector_stores/Neo4jVectorStore_test.py b/pkgs/community/tests/unit/vector_stores/Neo4jVectorStore_test.py index 74de4851b..d5e4699f6 100644 --- a/pkgs/community/tests/unit/vector_stores/Neo4jVectorStore_test.py +++ b/pkgs/community/tests/unit/vector_stores/Neo4jVectorStore_test.py @@ -2,7 +2,7 @@ import pytest from dotenv import load_dotenv from swarmauri.documents.concrete.Document import Document -from swarmauri_community.vector_stores.Neo4jVectorStore import Neo4jVectorStore +from swarmauri_community.vector_stores.concrete.Neo4jVectorStore import Neo4jVectorStore # Load environment variables load_dotenv() diff --git a/pkgs/community/tests/unit/vector_stores/PersistentChromadbVectorStore_test.py b/pkgs/community/tests/unit/vector_stores/PersistentChromadbVectorStore_test.py index b973277f1..66e0f0ed3 100644 --- a/pkgs/community/tests/unit/vector_stores/PersistentChromadbVectorStore_test.py +++ b/pkgs/community/tests/unit/vector_stores/PersistentChromadbVectorStore_test.py @@ -1,7 +1,7 @@ import os import pytest from swarmauri.documents.concrete.Document import Document -from swarmauri_community.vector_stores.PersistentChromaDBVectorStore import ( +from swarmauri_community.vector_stores.concrete.PersistentChromaDBVectorStore import ( PersistentChromaDBVectorStore, ) diff --git a/pkgs/community/tests/unit/vector_stores/PersistentQdrantVectorStore_test.py b/pkgs/community/tests/unit/vector_stores/PersistentQdrantVectorStore_test.py index 2277a87a4..d58c4295f 100644 --- a/pkgs/community/tests/unit/vector_stores/PersistentQdrantVectorStore_test.py +++ b/pkgs/community/tests/unit/vector_stores/PersistentQdrantVectorStore_test.py @@ -1,7 +1,7 @@ import os import pytest from swarmauri.documents.concrete.Document import Document -from swarmauri_community.vector_stores.PersistentQdrantVectorStore import ( +from swarmauri_community.vector_stores.concrete.PersistentQdrantVectorStore import ( PersistentQdrantVectorStore, ) diff --git a/pkgs/community/tests/unit/vector_stores/PineconeVectorStore_test.py b/pkgs/community/tests/unit/vector_stores/PineconeVectorStore_test.py index de2749df0..803a3f61c 100644 --- a/pkgs/community/tests/unit/vector_stores/PineconeVectorStore_test.py +++ b/pkgs/community/tests/unit/vector_stores/PineconeVectorStore_test.py @@ -1,7 +1,7 @@ import os import pytest from swarmauri.documents.concrete.Document import Document -from swarmauri_community.vector_stores.PineconeVectorStore import PineconeVectorStore +from swarmauri_community.vector_stores.concrete.PineconeVectorStore import PineconeVectorStore from dotenv import load_dotenv load_dotenv() diff --git a/pkgs/community/tests/unit/vector_stores/RedisVectorStore_test.py b/pkgs/community/tests/unit/vector_stores/RedisVectorStore_test.py index 17cbce8a2..80fc1a2d5 100644 --- a/pkgs/community/tests/unit/vector_stores/RedisVectorStore_test.py +++ b/pkgs/community/tests/unit/vector_stores/RedisVectorStore_test.py @@ -1,8 +1,6 @@ import pytest -import numpy as np -from swarmauri.documents.concrete.Document import Document -from swarmauri_community.vector_stores.RedisVectorStore import RedisVectorStore from swarmauri.documents.concrete.Document import Document +from swarmauri_community.vector_stores.concrete.RedisVectorStore import RedisVectorStore from dotenv import load_dotenv from os import getenv diff --git a/pkgs/swarmauri/swarmauri/utils/_lazy_import.py b/pkgs/swarmauri/swarmauri/utils/_lazy_import.py new file mode 100644 index 000000000..a3d3bd34a --- /dev/null +++ b/pkgs/swarmauri/swarmauri/utils/_lazy_import.py @@ -0,0 +1,22 @@ +import importlib + + +# Define a lazy loader function with a warning message if the module or class is not found +def _lazy_import(module_name, class_name): + try: + # Import the module + module = importlib.import_module(module_name) + # Dynamically get the class from the module + return getattr(module, class_name) + except ImportError: + # If module is not available, print a warning message + print( + f"Warning: The module '{module_name}' is not available. " + f"Please install the necessary dependencies to enable this functionality." + ) + return None + except AttributeError: + print( + f"Warning: The class '{class_name}' was not found in module '{module_name}'." + ) + return None From 58b6ad1fc4b6d685ba11a49b36eea1bfeac18939 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 21 Nov 2024 05:28:07 -0600 Subject: [PATCH 018/168] swarm - Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index 04eb35465..58608f84c 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "swarmauri" -version = "0.5.2" +version = "0.5.3.dev1" description = "This repository includes base classes, concrete generics, and concrete standard components within the Swarmauri framework." authors = ["Jacob Stewart "] license = "Apache-2.0" From 9e5376a8985b372bccf1bf0eeaff47ebeb4aba13 Mon Sep 17 00:00:00 2001 From: 3rdSon Date: Thu, 21 Nov 2024 12:35:23 +0100 Subject: [PATCH 019/168] swarm - changed the init files --- .../agent_factories/concrete/__init__.py | 30 +++++-- .../swarmauri/agents/concrete/__init__.py | 21 +---- .../swarmauri/chains/concrete/__init__.py | 22 +++-- .../swarmauri/chunkers/concrete/__init__.py | 26 +++--- .../conversations/concrete/__init__.py | 33 +++++--- .../swarmauri/distances/concrete/__init__.py | 53 ++++++------ .../swarmauri/documents/concrete/__init__.py | 12 ++- .../swarmauri/embeddings/concrete/__init__.py | 44 ++++------ .../swarmauri/exceptions/concrete/__init__.py | 14 +++- .../swarmauri/llms/concrete/__init__.py | 82 +++++++++---------- .../measurements/concrete/__init__.py | 47 +++++++++-- .../swarmauri/messages/concrete/__init__.py | 20 ++++- .../swarmauri/parsers/concrete/__init__.py | 58 ++++++------- .../swarmauri/prompts/concrete/__init__.py | 20 ++++- .../schema_converters/concrete/__init__.py | 58 +++++++------ .../swarmauri/swarms/concrete/__init__.py | 12 ++- .../swarmauri/toolkits/concrete/__init__.py | 19 +---- .../swarmauri/tools/concrete/__init__.py | 23 ++---- .../swarmauri/tracing/concrete/__init__.py | 22 +++-- .../swarmauri/swarmauri/utils/_lazy_import.py | 22 +++++ .../vector_stores/concrete/__init__.py | 32 +++----- .../swarmauri/vectors/concrete/__init__.py | 16 +++- 22 files changed, 383 insertions(+), 303 deletions(-) create mode 100644 pkgs/swarmauri/swarmauri/utils/_lazy_import.py diff --git a/pkgs/swarmauri/swarmauri/agent_factories/concrete/__init__.py b/pkgs/swarmauri/swarmauri/agent_factories/concrete/__init__.py index 651d9d992..8b75d563e 100644 --- a/pkgs/swarmauri/swarmauri/agent_factories/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/agent_factories/concrete/__init__.py @@ -1,8 +1,22 @@ -from swarmauri.agent_factories.concrete.agent_factory import AgentFactory -from swarmauri.agent_factories.concrete.conf_driven_agent_factory import ( - ConfDrivenAgentFactory, -) -from JsonAgentFactory import JsonAgentFactory -from swarmauri.agent_factories.concrete.ReflectionAgentFactory import ( - ReflectionAgentFactory, -) +from swarmauri.utils._lazy_import import _lazy_import + +# List of agent factory names (file names without the ".py" extension) and corresponding class names +agent_factory_files = [ + ("swarmauri.agent_factories.concrete.agent_factory", "AgentFactory"), + ( + "swarmauri.agent_factories.concrete.conf_driven_agent_factory", + "ConfDrivenAgentFactory", + ), + ("swarmauri.agent_factories.concrete.JsonAgentFactory", "JsonAgentFactory"), + ( + "swarmauri.agent_factories.concrete.ReflectionAgentFactory", + "ReflectionAgentFactory", + ), +] + +# Lazy loading of agent factories storing them in variables +for module_name, class_name in agent_factory_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded agent factories to __all__ +__all__ = [class_name for _, class_name in agent_factory_files] diff --git a/pkgs/swarmauri/swarmauri/agents/concrete/__init__.py b/pkgs/swarmauri/swarmauri/agents/concrete/__init__.py index f474dae0f..0103905cc 100644 --- a/pkgs/swarmauri/swarmauri/agents/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/agents/concrete/__init__.py @@ -1,27 +1,10 @@ -import importlib - -# Define a lazy loader function with a warning message if the module or class is not found -def _lazy_import(module_name, class_name): - try: - # Import the module - module = importlib.import_module(module_name) - # Dynamically get the class from the module - return getattr(module, class_name) - except ImportError: - # If module is not available, print a warning message - print(f"Warning: The module '{module_name}' is not available. " - f"Please install the necessary dependencies to enable this functionality.") - return None - except AttributeError: - # If class is not found, print a warning message - print(f"Warning: The class '{class_name}' was not found in module '{module_name}'.") - return None +from swarmauri.utils._lazy_import import _lazy_import # List of agent names (file names without the ".py" extension) and corresponding class names agent_files = [ - ("swarmauri.agents.concrete.SimpleConversationAgent", "SimpleConversationAgent"), ("swarmauri.agents.concrete.QAAgent", "QAAgent"), ("swarmauri.agents.concrete.RagAgent", "RagAgent"), + ("swarmauri.agents.concrete.SimpleConversationAgent", "SimpleConversationAgent"), ("swarmauri.agents.concrete.ToolAgent", "ToolAgent"), ] diff --git a/pkgs/swarmauri/swarmauri/chains/concrete/__init__.py b/pkgs/swarmauri/swarmauri/chains/concrete/__init__.py index efdd73eff..d6e508040 100644 --- a/pkgs/swarmauri/swarmauri/chains/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/chains/concrete/__init__.py @@ -1,11 +1,15 @@ -from swarmauri.chains.concrete.CallableChain import CallableChain -from swarmauri.chains.concrete.ChainStep import ChainStep -from swarmauri.chains.concrete.PromptContextChain import PromptContextChain -from swarmauri.chains.concrete.ContextChain import ContextChain +from swarmauri.utils._lazy_import import _lazy_import -__all__ = [ - "CallableChain", - "ChainStep", - "PromptContextChain", - "ContextChain", +chains_files = [ + ("swarmauri.chains.concrete.CallableChain import", "CallableChain"), + ("swarmauri.chains.concrete.ChainStep", "ChainStep"), + ("swarmauri.chains.concrete.PromptContextChain", "PromptContextChain"), + ("swarmauri.chains.concrete.ContextChain", "ContextChain"), ] + +# Lazy loading of chain classes, storing them in variables +for module_name, class_name in chains_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded chain classes to __all__ +__all__ = [class_name for _, class_name in chains_files] diff --git a/pkgs/swarmauri/swarmauri/chunkers/concrete/__init__.py b/pkgs/swarmauri/swarmauri/chunkers/concrete/__init__.py index f894163ad..fafead5cf 100644 --- a/pkgs/swarmauri/swarmauri/chunkers/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/chunkers/concrete/__init__.py @@ -1,13 +1,17 @@ -from swarmauri.chunkers.concrete.DelimiterBasedChunker import DelimiterBasedChunker -from swarmauri.chunkers.concrete.FixedLengthChunker import FixedLengthChunker -from swarmauri.chunkers.concrete.MdSnippetChunker import MdSnippetChunker -from swarmauri.chunkers.concrete.SentenceChunker import SentenceChunker -from swarmauri.chunkers.concrete.SlidingWindowChunker import SlidingWindowChunker +from swarmauri.utils._lazy_import import _lazy_import -__all__ = [ - "DelimiterBasedChunker", - "FixedLengthChunker", - "MdSnippetChunker", - "SentenceChunker", - "SlidingWindowChunker", +# List of chunker names (file names without the ".py" extension) and corresponding class names +chunkers_files = [ + ("swarmauri.chunkers.concrete.DelimiterBasedChunker", "DelimiterBasedChunker"), + ("swarmauri.chunkers.concrete.FixedLengthChunker", "FixedLengthChunker"), + ("swarmauri.chunkers.concrete.MdSnippetChunker", "MdSnippetChunker"), + ("swarmauri.chunkers.concrete.SentenceChunker", "SentenceChunker"), + ("swarmauri.chunkers.concrete.SlidingWindowChunker", "SlidingWindowChunker"), ] + +# Lazy loading of chunker classes, storing them in variables +for module_name, class_name in chunkers_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded chunker classes to __all__ +__all__ = [class_name for _, class_name in chunkers_files] diff --git a/pkgs/swarmauri/swarmauri/conversations/concrete/__init__.py b/pkgs/swarmauri/swarmauri/conversations/concrete/__init__.py index e51d24fe0..46179d6fb 100644 --- a/pkgs/swarmauri/swarmauri/conversations/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/conversations/concrete/__init__.py @@ -1,15 +1,22 @@ -from swarmauri.conversations.concrete.Conversation import Conversation -from swarmauri.conversations.concrete.MaxSystemContextConversation import ( - MaxSystemContextConversation, -) -from swarmauri.conversations.concrete.MaxSizeConversation import MaxSizeConversation -from swarmauri.conversations.concrete.SessionCacheConversation import ( - SessionCacheConversation, -) +from swarmauri.utils._lazy_import import _lazy_import -__all__ = [ - "Conversation", - "MaxSystemContextConversation", - "MaxSizeConversation", - "SessionCacheConversation", +# List of conversations names (file names without the ".py" extension) and corresponding class names +conversations_files = [ + ("swarmauri.conversations.concrete.Conversation", "Conversation"), + ( + "swarmauri.conversations.concrete.MaxSystemContextConversation", + "MaxSystemContextConversation", + ), + ("swarmauri.conversations.concrete.MaxSizeConversation", "MaxSizeConversation"), + ( + "swarmauri.conversations.concrete.SessionCacheConversation", + "SessionCacheConversation", + ), ] + +# Lazy loading of conversations classes, storing them in variables +for module_name, class_name in conversations_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded conversations classes to __all__ +__all__ = [class_name for _, class_name in conversations_files] diff --git a/pkgs/swarmauri/swarmauri/distances/concrete/__init__.py b/pkgs/swarmauri/swarmauri/distances/concrete/__init__.py index 9e163ca4d..033a0dd13 100644 --- a/pkgs/swarmauri/swarmauri/distances/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/distances/concrete/__init__.py @@ -1,34 +1,27 @@ -import importlib +from swarmauri.utils._lazy_import import _lazy_import -# Define a lazy loader function with a warning message if the module is not found -def _lazy_import(module_name, module_description=None): - try: - return importlib.import_module(module_name) - except ImportError: - # If module is not available, print a warning message - print(f"Warning: The module '{module_description or module_name}' is not available. " - f"Please install the necessary dependencies to enable this functionality.") - return None - -# List of distance names (file names without the ".py" extension) -distance_files = [ - "CanberraDistance", - "ChebyshevDistance", - "ChiSquaredDistance", - "CosineDistance", - "EuclideanDistance", - "HaversineDistance", - "JaccardIndexDistance", - "LevenshteinDistance", - "ManhattanDistance", - "MinkowskiDistance", - "SorensenDiceDistance", - "SquaredEuclideanDistance", +# List of distances names (file names without the ".py" extension) and corresponding class names +distances_files = [ + ("swarmauri.distances.concrete.CanberraDistance", "CanberraDistance"), + ("swarmauri.distances.concrete.ChebyshevDistance", "ChebyshevDistance"), + ("swarmauri.distances.concrete.ChiSquaredDistance", "ChiSquaredDistance"), + ("swarmauri.distances.concrete.CosineDistance", "CosineDistance"), + ("swarmauri.distances.concrete.EuclideanDistance", "EuclideanDistance"), + ("swarmauri.distances.concrete.HaversineDistance", "HaversineDistance"), + ("swarmauri.distances.concrete.JaccardIndexDistance", "JaccardIndexDistance"), + ("swarmauri.distances.concrete.LevenshteinDistance", "LevenshteinDistance"), + ("swarmauri.distances.concrete.ManhattanDistance", "ManhattanDistance"), + ("swarmauri.distances.concrete.MinkowskiDistance", "MinkowskiDistance"), + ("swarmauri.distances.concrete.SorensenDiceDistance", "SorensenDiceDistance"), + ( + "swarmauri.distances.concrete.SquaredEuclideanDistance", + "SquaredEuclideanDistance", + ), ] -# Lazy loading of distance modules, storing them in variables -for distance in distance_files: - globals()[distance] = _lazy_import(f"swarmauri.distances.concrete.{distance}", distance) +# Lazy loading of distances classes, storing them in variables +for module_name, class_name in distances_files: + globals()[class_name] = _lazy_import(module_name, class_name) -# Adding the lazy-loaded distance modules to __all__ -__all__ = distance_files +# Adding the lazy-loaded distances classes to __all__ +__all__ = [class_name for _, class_name in distances_files] diff --git a/pkgs/swarmauri/swarmauri/documents/concrete/__init__.py b/pkgs/swarmauri/swarmauri/documents/concrete/__init__.py index f0725fde0..c4b50e1a5 100644 --- a/pkgs/swarmauri/swarmauri/documents/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/documents/concrete/__init__.py @@ -1 +1,11 @@ -from swarmauri.documents.concrete import * +from swarmauri.utils._lazy_import import _lazy_import + +# List of documents names (file names without the ".py" extension) and corresponding class names +documents_files = [("swarmauri.documents.concrete.Document", "Document")] + +# Lazy loading of documents classes, storing them in variables +for module_name, class_name in documents_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded documents classes to __all__ +__all__ = [class_name for _, class_name in documents_files] diff --git a/pkgs/swarmauri/swarmauri/embeddings/concrete/__init__.py b/pkgs/swarmauri/swarmauri/embeddings/concrete/__init__.py index a4fd73974..c6d12f871 100644 --- a/pkgs/swarmauri/swarmauri/embeddings/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/embeddings/concrete/__init__.py @@ -1,31 +1,19 @@ -import importlib +from swarmauri.utils._lazy_import import _lazy_import -# Define a lazy loader function with a warning message if the module is not found -def _lazy_import(module_name, module_description=None): - try: - return importlib.import_module(module_name) - except ImportError: - # If module is not available, print a warning message - print(f"Warning: The module '{module_description or module_name}' is not available. " - f"Please install the necessary dependencies to enable this functionality.") - return None +# List of embeddings names (file names without the ".py" extension) and corresponding class names +embeddings_files = [ + ("swarmauri.embeddings.concrete.CohereEmbedding", "CohereEmbedding"), + ("swarmauri.embeddings.concrete.GeminiEmbedding", "GeminiEmbedding"), + ("swarmauri.embeddings.concrete.MistralEmbedding", "MistralEmbedding"), + ("swarmauri.embeddings.concrete.NmfEmbedding", "NmfEmbedding"), + ("swarmauri.embeddings.concrete.OpenAIEmbedding", "OpenAIEmbedding"), + ("swarmauri.embeddings.concrete.TfidfEmbedding", "TfidfEmbedding"), + ("swarmauri.embeddings.concrete.VoyageEmbedding", "VoyageEmbedding"), +] -# Lazy loading of embeddings with descriptive names -# Doc2VecEmbedding = _lazy_import("swarmauri.embeddings.concrete.Doc2VecEmbedding", "Doc2VecEmbedding") -GeminiEmbedding = _lazy_import("swarmauri.embeddings.concrete.GeminiEmbedding", "GeminiEmbedding") -MistralEmbedding = _lazy_import("swarmauri.embeddings.concrete.MistralEmbedding", "MistralEmbedding") -# MlmEmbedding = _lazy_import("swarmauri.embeddings.concrete.MlmEmbedding", "MlmEmbedding") -NmfEmbedding = _lazy_import("swarmauri.embeddings.concrete.NmfEmbedding", "NmfEmbedding") -OpenAIEmbedding = _lazy_import("swarmauri.embeddings.concrete.OpenAIEmbedding", "OpenAIEmbedding") -TfidfEmbedding = _lazy_import("swarmauri.embeddings.concrete.TfidfEmbedding", "TfidfEmbedding") +# Lazy loading of embeddings classes, storing them in variables +for module_name, class_name in embeddings_files: + globals()[class_name] = _lazy_import(module_name, class_name) -# Adding lazy-loaded modules to __all__ -__all__ = [ - # "Doc2VecEmbedding", - "GeminiEmbedding", - "MistralEmbedding", - # "MlmEmbedding", - "NmfEmbedding", - "OpenAIEmbedding", - "TfidfEmbedding", -] +# Adding the lazy-loaded embeddings classes to __all__ +__all__ = [class_name for _, class_name in embeddings_files] diff --git a/pkgs/swarmauri/swarmauri/exceptions/concrete/__init__.py b/pkgs/swarmauri/swarmauri/exceptions/concrete/__init__.py index 43b631bc1..2baf7a56d 100644 --- a/pkgs/swarmauri/swarmauri/exceptions/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/exceptions/concrete/__init__.py @@ -1,3 +1,13 @@ -from swarmauri.exceptions.concrete.IndexErrorWithContext import IndexErrorWithContext +from swarmauri.utils._lazy_import import _lazy_import -__all__ = ["IndexErrorWithContext"] +# List of exceptions names (file names without the ".py" extension) and corresponding class names +exceptions_files = [ + ("swarmauri.exceptions.concrete.IndexErrorWithContext", "IndexErrorWithContext"), +] + +# Lazy loading of exceptions classes, storing them in variables +for module_name, class_name in exceptions_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded exceptions classes to __all__ +__all__ = [class_name for _, class_name in exceptions_files] diff --git a/pkgs/swarmauri/swarmauri/llms/concrete/__init__.py b/pkgs/swarmauri/swarmauri/llms/concrete/__init__.py index a24e7b59f..975ac7e93 100644 --- a/pkgs/swarmauri/swarmauri/llms/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/llms/concrete/__init__.py @@ -1,47 +1,43 @@ -import importlib +from swarmauri.utils._lazy_import import _lazy_import -# Define a lazy loader function with a warning message if the module is not found -def _lazy_import(module_name, module_description=None): - try: - return importlib.import_module(module_name) - except ImportError: - # If module is not available, print a warning message - print(f"Warning: The module '{module_description or module_name}' is not available. " - f"Please install the necessary dependencies to enable this functionality.") - return None - -# List of model names (file names without the ".py" extension) -model_files = [ - "AI21StudioModel", - "AnthropicModel", - "AnthropicToolModel", - "BlackForestimgGenModel", - "CohereModel", - "CohereToolModel", - "DeepInfraImgGenModel", - "DeepInfraModel", - "DeepSeekModel", - "FalAllImgGenModel", - "FalAVisionModel", - "GeminiProModel", - "GeminiToolModel", - "GroqAudio", - "GroqModel", - "GroqToolModel", - "GroqVisionModel", - "MistralModel", - "MistralToolModel", - "OpenAIGenModel", - "OpenAIModel", - "OpenAIToolModel", - "PerplexityModel", - "PlayHTModel", - "WhisperLargeModel", +# List of llms names (file names without the ".py" extension) and corresponding class names +llms_files = [ + ("swarmauri.llms.concrete.AI21StudioModel", "AI21StudioModel"), + ("swarmauri.llms.concrete.AnthropicModel", "AnthropicModel"), + ("swarmauri.llms.concrete.AnthropicToolModel", "AnthropicToolModel"), + ("swarmauri.llms.concrete.BlackForestImgGenModel", "BlackForestImgGenModel"), + ("swarmauri.llms.concrete.CohereModel", "CohereModel"), + ("swarmauri.llms.concrete.CohereToolModel", "CohereToolModel"), + ("swarmauri.llms.concrete.DeepInfraImgGenModel", "DeepInfraImgGenModel"), + ("swarmauri.llms.concrete.DeepInfraModel", "DeepInfraModel"), + ("swarmauri.llms.concrete.DeepSeekModel", "DeepSeekModel"), + ("swarmauri.llms.concrete.FalAIImgGenModel", "FalaiImgGenModel"), + ("swarmauri.llms.concrete.FalAIVisionModel", "FalAIVisionModel"), + ("swarmauri.llms.concrete.GeminiProModel", "GeminiProModel"), + ("swarmauri.llms.concrete.GeminiToolModel", "GeminiToolModel"), + ("swarmauri.llms.concrete.GroqAIAudio", "GroqAIAudio"), + ("swarmauri.llms.concrete.GroqModel", "GroqModel"), + ("swarmauri.llms.concrete.GroqToolModel", "GroqToolModel"), + ("swarmauri.llms.concrete.GroqVisionModel", "GroqVisionModel"), + ("swarmauri.llms.concrete.HyperbolicAudioTTS", "HyperbolicAudioTTS"), + ("swarmauri.llms.concrete.HyperbolicImgGenModel", "HyperbolicImgGenModel"), + ("swarmauri.llms.concrete.HyperbolicModel", "HyperbolicModel"), + ("swarmauri.llms.concrete.HyperbolicVisionModel", "HyperbolicVisionModel"), + ("swarmauri.llms.concrete.MistralModel", "MistralModel"), + ("swarmauri.llms.concrete.MistralToolModel", "MistralToolModel"), + ("swarmauri.llms.concrete.OpenAIAudio", "OpenAIAudio"), + ("swarmauri.llms.concrete.OpenAIAudioTTS", "OpenAIAudioTTS"), + ("swarmauri.llms.concrete.OpenAIImgGenModel", "OpenAIImgGenModel"), + ("swarmauri.llms.concrete.OpenAIModel", "OpenAIModel"), + ("swarmauri.llms.concrete.OpenAIToolModel", "OpenAIToolModel"), + ("swarmauri.llms.concrete.PerplexityModel", "PerplexityModel"), + ("swarmauri.llms.concrete.PlayHTModel", "PlayHTModel"), + ("swarmauri.llms.concrete.WhisperLargeModel", "WhisperLargeModel"), ] -# Lazy loading of models, storing them in variables -for model in model_files: - globals()[model] = _lazy_import(f"swarmauri.llms.concrete.{model}", model) +# Lazy loading of llms classes, storing them in variables +for module_name, class_name in llms_files: + globals()[class_name] = _lazy_import(module_name, class_name) -# Adding the lazy-loaded models to __all__ -__all__ = model_files +# Adding the lazy-loaded llms classes to __all__ +__all__ = [class_name for _, class_name in llms_files] diff --git a/pkgs/swarmauri/swarmauri/measurements/concrete/__init__.py b/pkgs/swarmauri/swarmauri/measurements/concrete/__init__.py index e340b2b85..ea47cb17f 100644 --- a/pkgs/swarmauri/swarmauri/measurements/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/measurements/concrete/__init__.py @@ -1,6 +1,41 @@ -from swarmauri.measurements.concrete.FirstImpressionMeasurement import FirstImpressionMeasurement -from swarmauri.measurements.concrete.MeanMeasurement import MeanMeasurement -from swarmauri.measurements.concrete.PatternMatchingMeasurement import PatternMatchingMeasurement -from swarmauri.measurements.concrete.RatioOfSumsMeasurement import RatioOfSumsMeasurement -from swarmauri.measurements.concrete.StaticMeasurement import StaticMeasurement -from swarmauri.measurements.concrete.ZeroMeasurement import ZeroMeasurement +from swarmauri.utils._lazy_import import _lazy_import + +# List of measurements names (file names without the ".py" extension) and corresponding class names +measurements_files = [ + ( + "swarmauri.measurements.concrete.CompletenessMeasurement", + "CompletenessMeasurement", + ), + ( + "swarmauri.measurements.concrete.DistinctivenessMeasurement", + "DistinctivenessMeasurement", + ), + ( + "swarmauri.measurements.concrete.FirstImpressionMeasurement", + "FirstImpressionMeasurement", + ), + ("swarmauri.measurements.concrete.MeanMeasurement", "MeanMeasurement"), + ("swarmauri.measurements.concrete.MiscMeasurement", "MiscMeasurement"), + ( + "swarmauri.measurements.concrete.MissingnessMeasurement", + "MissingnessMeasurement", + ), + ( + "swarmauri.measurements.concrete.PatternMatchingMeasurement", + "PatternMatchingMeasurement", + ), + ( + "swarmauri.measurements.concrete.RatioOfSumsMeasurement", + "RatioOfSumsMeasurement", + ), + ("swarmauri.measurements.concrete.StaticMeasurement", "StaticMeasurement"), + ("swarmauri.measurements.concrete.UniquenessMeasurement", "UniquenessMeasurement"), + ("swarmauri.measurements.concrete.ZeroMeasurement", "ZeroMeasurement"), +] + +# Lazy loading of measurements classes, storing them in variables +for module_name, class_name in measurements_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded measurements classes to __all__ +__all__ = [class_name for _, class_name in measurements_files] diff --git a/pkgs/swarmauri/swarmauri/messages/concrete/__init__.py b/pkgs/swarmauri/swarmauri/messages/concrete/__init__.py index 5c619ecc8..716bd57c5 100644 --- a/pkgs/swarmauri/swarmauri/messages/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/messages/concrete/__init__.py @@ -1,4 +1,16 @@ -from swarmauri.messages.concrete.HumanMessage import HumanMessage -from swarmauri.messages.concrete.AgentMessage import AgentMessage -from swarmauri.messages.concrete.FunctionMessage import FunctionMessage -from swarmauri.messages.concrete.SystemMessage import SystemMessage +from swarmauri.utils._lazy_import import _lazy_import + +# List of messages names (file names without the ".py" extension) and corresponding class names +messages_files = [ + ("swarmauri.messages.concrete.HumanMessage", "HumanMessage"), + ("swarmauri.messages.concrete.AgentMessage", "AgentMessage"), + ("from swarmauri.messages.concrete.FunctionMessage", "FunctionMessage"), + ("swarmauri.messages.concrete.SystemMessage", "SystemMessage"), +] + +# Lazy loading of messages classes, storing them in variables +for module_name, class_name in messages_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded messages classes to __all__ +__all__ = [class_name for _, class_name in messages_files] diff --git a/pkgs/swarmauri/swarmauri/parsers/concrete/__init__.py b/pkgs/swarmauri/swarmauri/parsers/concrete/__init__.py index a42b1b55f..fb730763f 100644 --- a/pkgs/swarmauri/swarmauri/parsers/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/parsers/concrete/__init__.py @@ -1,37 +1,29 @@ -import importlib +from swarmauri.utils._lazy_import import _lazy_import -# Define a lazy loader function with a warning message if the module is not found -def _lazy_import(module_name, module_description=None): - try: - return importlib.import_module(module_name) - except ImportError: - # If module is not available, print a warning message - print(f"Warning: The module '{module_description or module_name}' is not available. " - f"Please install the necessary dependencies to enable this functionality.") - return None - -# List of parser names (file names without the ".py" extension) -parser_files = [ - "BeautifulSoupElementParser", - # "BERTEmbeddingParser", - "CSVParser", - "EntityRecognitionParser", - "HTMLTagStripParser", - "KeywordExtractorParser", - "Md2HtmlParser", - "OpenAPISpecParser", - "PhoneNumberExtractorParser", - "PythonParser", - "RegExParser", - # "TextBlobNounParser", - # "TextBlobSentenceParser", - "URLExtractorParser", - "XMLParser", +# List of parsers names (file names without the ".py" extension) and corresponding class names +parsers_files = [ + ( + "swarmauri.parsers.concrete.BeautifulSoupElementParser", + "BeautifulSoupElementParser", + ), + ("swarmauri.parsers.concrete.CSVParser", "CSVParser"), + ("swarmauri.parsers.concrete.HTMLTagStripParser", "HTMLTagStripParser"), + ("swarmauri.parsers.concrete.KeywordExtractorParser", "KeywordExtractorParser"), + ("swarmauri.parsers.concrete.Md2HtmlParser", "Md2HtmlParser"), + ("swarmauri.parsers.concrete.OpenAPISpecParser", "OpenAPISpecParser"), + ( + "swarmauri.parsers.concrete.PhoneNumberExtractorParser", + "PhoneNumberExtractorParser", + ), + ("swarmauri.parsers.concrete.PythonParser", "PythonParser"), + ("swarmauri.parsers.concrete.RegExParser", "RegExParser"), + ("swarmauri.parsers.concrete.URLExtractorParser", "URLExtractorParser"), + ("swarmauri.parsers.concrete.XMLParser", "XMLParser"), ] -# Lazy loading of parser modules, storing them in variables -for parser in parser_files: - globals()[parser] = _lazy_import(f"swarmauri.parsers.concrete.{parser}", parser) +# Lazy loading of parsers classes, storing them in variables +for module_name, class_name in parsers_files: + globals()[class_name] = _lazy_import(module_name, class_name) -# Adding the lazy-loaded parser modules to __all__ -__all__ = parser_files +# Adding the lazy-loaded parsers classes to __all__ +__all__ = [class_name for _, class_name in parsers_files] diff --git a/pkgs/swarmauri/swarmauri/prompts/concrete/__init__.py b/pkgs/swarmauri/swarmauri/prompts/concrete/__init__.py index 00d6b3cb9..3755b609f 100644 --- a/pkgs/swarmauri/swarmauri/prompts/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/prompts/concrete/__init__.py @@ -1,4 +1,16 @@ -from swarmauri.prompts.concrete.Prompt import Prompt -from swarmauri.prompts.concrete.PromptGenerator import PromptGenerator -from swarmauri.prompts.concrete.PromptMatrix import PromptMatrix -from swarmauri.prompts.concrete.PromptTemplate import PromptTemplate +from swarmauri.utils._lazy_import import _lazy_import + +# List of prompts names (file names without the ".py" extension) and corresponding class names +prompts_files = [ + ("swarmauri.prompts.concrete.Prompt", "Prompt"), + ("swarmauri.prompts.concrete.PromptGenerator", "PromptGenerator"), + ("swarmauri.prompts.concrete.PromptMatrix", "PromptMatrix"), + ("from swarmauri.prompts.concrete.PromptTemplate", "PromptTemplate"), +] + +# Lazy loading of prompts classes, storing them in variables +for module_name, class_name in prompts_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded prompts classes to __all__ +__all__ = [class_name for _, class_name in prompts_files] diff --git a/pkgs/swarmauri/swarmauri/schema_converters/concrete/__init__.py b/pkgs/swarmauri/swarmauri/schema_converters/concrete/__init__.py index c608d8c11..65044d64d 100644 --- a/pkgs/swarmauri/swarmauri/schema_converters/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/schema_converters/concrete/__init__.py @@ -1,29 +1,37 @@ -import importlib +from swarmauri.utils._lazy_import import _lazy_import -# Define a lazy loader function with a warning message if the module is not found -def _lazy_import(module_name, module_description=None): - try: - return importlib.import_module(module_name) - except ImportError: - # If module is not available, print a warning message - print(f"Warning: The module '{module_description or module_name}' is not available. " - f"Please install the necessary dependencies to enable this functionality.") - return None - -# List of schema converter names (file names without the ".py" extension) -schema_converter_files = [ - "AnthropicSchemaConverter", - "CohereSchemaConverter", - "GeminiSchemaConverter", - "GroqSchemaConverter", - "MistralSchemaConverter", - "OpenAISchemaConverter", - "ShuttleAISchemaConverter", +# List of schema_converters names (file names without the ".py" extension) and corresponding class names +schema_converters_files = [ + ( + "swarmauri.schema_converters.concrete.AnthropicSchemaConverter", + "AnthropicSchemaConverter", + ), + ( + "swarmauri.schema_converters.concrete.CohereSchemaConverter", + "CohereSchemaConverter", + ), + ( + "swarmauri.schema_converters.concrete.GeminiSchemaConverter", + "GeminiSchemaConverter", + ), + ("swarmauri.schema_converters.concrete.GroqSchemaConverter", "GroqSchemaConverter"), + ( + "swarmauri.schema_converters.concrete.MistralSchemaConverter", + "MistralSchemaConverter", + ), + ( + "swarmauri.schema_converters.concrete.OpenAISchemaConverter", + "OpenAISchemaConverter", + ), + ( + "swarmauri.schema_converters.concrete.ShuttleAISchemaConverter", + "ShuttleAISchemaConverter", + ), ] -# Lazy loading of schema converters, storing them in variables -for schema_converter in schema_converter_files: - globals()[schema_converter] = _lazy_import(f"swarmauri.schema_converters.concrete.{schema_converter}", schema_converter) +# Lazy loading of schema_converters classes, storing them in variables +for module_name, class_name in schema_converters_files: + globals()[class_name] = _lazy_import(module_name, class_name) -# Adding the lazy-loaded schema converters to __all__ -__all__ = schema_converter_files +# Adding the lazy-loaded schema_converters classes to __all__ +__all__ = [class_name for _, class_name in schema_converters_files] diff --git a/pkgs/swarmauri/swarmauri/swarms/concrete/__init__.py b/pkgs/swarmauri/swarmauri/swarms/concrete/__init__.py index bd32d1999..61f84eae6 100644 --- a/pkgs/swarmauri/swarmauri/swarms/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/swarms/concrete/__init__.py @@ -1 +1,11 @@ -from swarmauri.swarms.concrete.SimpleSwarmFactory import SimpleSwarmFactory +from swarmauri.utils._lazy_import import _lazy_import + +# List of swarms names (file names without the ".py" extension) and corresponding class names +swarms_files = [("swarmauri.swarms.concrete.SimpleSwarmFactory", "SimpleSwarmFactory")] + +# Lazy loading of swarms classes, storing them in variables +for module_name, class_name in swarms_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded swarms classes to __all__ +__all__ = [class_name for _, class_name in swarms_files] diff --git a/pkgs/swarmauri/swarmauri/toolkits/concrete/__init__.py b/pkgs/swarmauri/swarmauri/toolkits/concrete/__init__.py index 87127d6bf..a7311c7c9 100644 --- a/pkgs/swarmauri/swarmauri/toolkits/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/toolkits/concrete/__init__.py @@ -1,21 +1,4 @@ -import importlib - -# Define a lazy loader function with a warning message if the module or class is not found -def _lazy_import(module_name, class_name): - try: - # Import the module - module = importlib.import_module(module_name) - # Dynamically get the class from the module - return getattr(module, class_name) - except ImportError: - # If module is not available, print a warning message - print(f"Warning: The module '{module_name}' is not available. " - f"Please install the necessary dependencies to enable this functionality.") - return None - except AttributeError: - # If class is not found, print a warning message - print(f"Warning: The class '{class_name}' was not found in module '{module_name}'.") - return None +from swarmauri.utils._lazy_import import _lazy_import # List of toolkit names (file names without the ".py" extension) and corresponding class names toolkit_files = [ diff --git a/pkgs/swarmauri/swarmauri/tools/concrete/__init__.py b/pkgs/swarmauri/swarmauri/tools/concrete/__init__.py index f9d2a297e..5b7d61054 100644 --- a/pkgs/swarmauri/swarmauri/tools/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/tools/concrete/__init__.py @@ -1,25 +1,12 @@ -import importlib - -# Define a lazy loader function with a warning message if the module or class is not found -def _lazy_import(module_name, class_name): - try: - # Import the module - module = importlib.import_module(module_name) - # Dynamically get the class from the module - return getattr(module, class_name) - except ImportError: - # If module is not available, print a warning message - print(f"Warning: The module '{module_name}' is not available. " - f"Please install the necessary dependencies to enable this functionality.") - return None - except AttributeError: - print(f"Warning: The class '{class_name}' was not found in module '{module_name}'.") - return None +from swarmauri.utils._lazy_import import _lazy_import # List of tool names (file names without the ".py" extension) and corresponding class names tool_files = [ ("swarmauri.tools.concrete.AdditionTool", "AdditionTool"), - ("swarmauri.tools.concrete.AutomatedReadabilityIndexTool", "AutomatedReadabilityIndexTool"), + ( + "swarmauri.tools.concrete.AutomatedReadabilityIndexTool", + "AutomatedReadabilityIndexTool", + ), ("swarmauri.tools.concrete.CalculatorTool", "CalculatorTool"), ("swarmauri.tools.concrete.CodeExtractorTool", "CodeExtractorTool"), ("swarmauri.tools.concrete.CodeInterpreterTool", "CodeInterpreterTool"), diff --git a/pkgs/swarmauri/swarmauri/tracing/concrete/__init__.py b/pkgs/swarmauri/swarmauri/tracing/concrete/__init__.py index 95900d024..1b6619352 100644 --- a/pkgs/swarmauri/swarmauri/tracing/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/tracing/concrete/__init__.py @@ -1,5 +1,17 @@ -from swarmauri.tracing.concrete.CallableTracer import CallableTracer -from swarmauri.tracing.concrete.ChainTracer import ChainTracer -from swarmauri.tracing.concrete.SimpleTraceContext import SimpleTraceContext -from swarmauri.tracing.concrete.TracedVariable import TracedVariable -from swarmauri.tracing.concrete.VariableTracer import VariableTracer +from swarmauri.utils._lazy_import import _lazy_import + +# List of tracing names (file names without the ".py" extension) and corresponding class names +tracing_files = [ + ("swarmauri.tracing.concrete.CallableTracer", "CallableTracer"), + ("from swarmauri.tracing.concrete.ChainTracer", "ChainTracer"), + ("swarmauri.tracing.concrete.SimpleTraceContext", "SimpleTraceContext"), + ("swarmauri.tracing.concrete.TracedVariable", "TracedVariable"), + ("swarmauri.tracing.concrete.VariableTracer", "VariableTracer"), +] + +# Lazy loading of tracings, storing them in variables +for module_name, class_name in tracing_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded tracings to __all__ +__all__ = [class_name for _, class_name in tracing_files] diff --git a/pkgs/swarmauri/swarmauri/utils/_lazy_import.py b/pkgs/swarmauri/swarmauri/utils/_lazy_import.py new file mode 100644 index 000000000..a3d3bd34a --- /dev/null +++ b/pkgs/swarmauri/swarmauri/utils/_lazy_import.py @@ -0,0 +1,22 @@ +import importlib + + +# Define a lazy loader function with a warning message if the module or class is not found +def _lazy_import(module_name, class_name): + try: + # Import the module + module = importlib.import_module(module_name) + # Dynamically get the class from the module + return getattr(module, class_name) + except ImportError: + # If module is not available, print a warning message + print( + f"Warning: The module '{module_name}' is not available. " + f"Please install the necessary dependencies to enable this functionality." + ) + return None + except AttributeError: + print( + f"Warning: The class '{class_name}' was not found in module '{module_name}'." + ) + return None diff --git a/pkgs/swarmauri/swarmauri/vector_stores/concrete/__init__.py b/pkgs/swarmauri/swarmauri/vector_stores/concrete/__init__.py index 2f946b377..ceb2b245c 100644 --- a/pkgs/swarmauri/swarmauri/vector_stores/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/vector_stores/concrete/__init__.py @@ -1,26 +1,14 @@ -import importlib +from swarmauri.utils._lazy_import import _lazy_import -# Define a lazy loader function with a warning message if the module is not found -def _lazy_import(module_name, module_description=None): - try: - return importlib.import_module(module_name) - except ImportError: - # If module is not available, print a warning message - print(f"Warning: The module '{module_description or module_name}' is not available. " - f"Please install the necessary dependencies to enable this functionality.") - return None - -# List of vector store names (file names without the ".py" extension) -vector_store_files = [ - # "Doc2VecVectorStore", - # "MlmVectorStore", - "SqliteVectorStore", - "TfidfVectorStore", +# List of vectore_stores names (file names without the ".py" extension) and corresponding class names +vectore_stores_files = [ + ("swarmauri.vector_stores.concrete.SqliteVectorStore", "SqliteVectorStore"), + ("swarmauri.vector_stores.concrete.TfidfVectorStore", "TfidfVectorStore"), ] -# Lazy loading of vector stores, storing them in variables -for vector_store in vector_store_files: - globals()[vector_store] = _lazy_import(f"swarmauri.vector_stores.concrete.{vector_store}", vector_store) +# Lazy loading of vectore_storess, storing them in variables +for module_name, class_name in vectore_stores_files: + globals()[class_name] = _lazy_import(module_name, class_name) -# Adding the lazy-loaded vector stores to __all__ -__all__ = vector_store_files +# Adding the lazy-loaded vectore_storess to __all__ +__all__ = [class_name for _, class_name in vectore_stores_files] diff --git a/pkgs/swarmauri/swarmauri/vectors/concrete/__init__.py b/pkgs/swarmauri/swarmauri/vectors/concrete/__init__.py index 16f348f20..7283bc0a9 100644 --- a/pkgs/swarmauri/swarmauri/vectors/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/vectors/concrete/__init__.py @@ -1,4 +1,14 @@ -# -*- coding: utf-8 -*- +from swarmauri.utils._lazy_import import _lazy_import -from swarmauri.vectors.concrete.Vector import Vector -from swarmauri.vectors.concrete.VectorProductMixin import VectorProductMixin +# List of vectors names (file names without the ".py" extension) and corresponding class names +vectors_files = [ + ("swarmauri.vectors.concrete.Vector", "Vector"), + ("swarmauri.vectors.concrete.VectorProductMixin", "VectorProductMixin"), +] + +# Lazy loading of vectorss, storing them in variables +for module_name, class_name in vectors_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded vectorss to __all__ +__all__ = [class_name for _, class_name in vectors_files] From 4823f078738e231651bdc1ce17df7c38d8a4624b Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 21 Nov 2024 05:38:25 -0600 Subject: [PATCH 020/168] comm - Update pyproject.toml --- pkgs/community/pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/community/pyproject.toml b/pkgs/community/pyproject.toml index 332deb6bd..a2eb33d1c 100644 --- a/pkgs/community/pyproject.toml +++ b/pkgs/community/pyproject.toml @@ -15,7 +15,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.13" -swarmauri = "==0.5.2" +swarmauri = "==0.5.3.dev1" typing_extensions = "*" matplotlib = { version = ">=3.9.2", optional = true } nltk = { version = "^3.9.1", optional = true } @@ -88,4 +88,4 @@ log_cli = true log_cli_level = "INFO" log_cli_format = "%(asctime)s [%(levelname)s] %(message)s" log_cli_date_format = "%Y-%m-%d %H:%M:%S" -asyncio_default_fixture_loop_scope = "function" \ No newline at end of file +asyncio_default_fixture_loop_scope = "function" From d7754930db05f18807d524a2b86a98bc87a88942 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 21 Nov 2024 05:42:21 -0600 Subject: [PATCH 021/168] swarm - Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index 58608f84c..27656153a 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "swarmauri" -version = "0.5.3.dev1" +version = "0.5.3.dev2" description = "This repository includes base classes, concrete generics, and concrete standard components within the Swarmauri framework." authors = ["Jacob Stewart "] license = "Apache-2.0" @@ -52,7 +52,11 @@ matplotlib = { version = ">=3.9.2", optional = true } # Extras without versioning, grouped for specific use cases io = ["aiofiles", "aiohttp"] #llms = ["cohere", "mistralai", "fal-client", "google-generativeai", "openai"] -nlp = ["nltk", "textblob", "yake"] +nlp = [ + #"nltk", + #"textblob", + "yake" +] nlp_tools = ["beautifulsoup4"] #ml_toolkits = ["gensim", "scipy", "scikit-learn"] #spacy = ["spacy"] From b2da5246038e50b8916779695c684e099b593daa Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 21 Nov 2024 05:43:14 -0600 Subject: [PATCH 022/168] cicd - Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index 2f28a48fd..fbe485428 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -126,7 +126,7 @@ jobs: - name: Install package dependencies run: | cd pkgs/${{ matrix.package_tests.package }} - poetry install --no-cache -vv + poetry install --no-cache --all-extras -vv - name: Run all tests for the package run: | From 80821d3a909606f9f2324c09031eb4a407874e3e Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 21 Nov 2024 05:48:47 -0600 Subject: [PATCH 023/168] Create increment_version_dev.yaml --- .github/workflows/increment_version_dev.yaml | 85 ++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 .github/workflows/increment_version_dev.yaml diff --git a/.github/workflows/increment_version_dev.yaml b/.github/workflows/increment_version_dev.yaml new file mode 100644 index 000000000..551b83093 --- /dev/null +++ b/.github/workflows/increment_version_dev.yaml @@ -0,0 +1,85 @@ +name: Increment Versions + +on: + workflow_dispatch: + +jobs: + increment-versions: + runs-on: ubuntu-latest + + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install TOML editor + run: | + python -m pip install --upgrade pip + pip install tomlkit + + - name: Increment versions in pyproject.toml + run: | + echo "Incrementing versions..." + find . -name "pyproject.toml" | while read -r pyproject; do + echo "Processing $pyproject" + + # Extract current version + CURRENT_VERSION=$(python -c " +import tomlkit +with open('$pyproject', 'r') as f: + data = tomlkit.parse(f.read()) + print(data['tool']['poetry']['version']) +") + + # Increment version + BASE_VERSION=$(echo "$CURRENT_VERSION" | sed -E 's/(.*)-dev.*/\1/') + DEV_PART=$(echo "$CURRENT_VERSION" | grep -oE 'dev[0-9]+$' | grep -oE '[0-9]+') + NEW_DEV_PART=$((DEV_PART + 1)) + NEW_VERSION="${BASE_VERSION}-dev${NEW_DEV_PART:-1}" + + echo "Updating version from $CURRENT_VERSION to $NEW_VERSION" + + # Update version in pyproject.toml + python -c " +import tomlkit +with open('$pyproject', 'r') as f: + data = tomlkit.parse(f.read()) +data['tool']['poetry']['version'] = '$NEW_VERSION' +with open('$pyproject', 'w') as f: + f.write(tomlkit.dumps(data)) +" + + # Update dependencies starting with 'swarmauri' + python -c " +import tomlkit +with open('$pyproject', 'r') as f: + data = tomlkit.parse(f.read()) +dependencies = data['tool']['poetry'].get('dependencies', {}) +for dep, version in dependencies.items(): + if dep.startswith('swarmauri') and isinstance(version, str): + base_version = version.split('-dev')[0] + dev_part = int(version.split('-dev')[-1]) if '-dev' in version else 0 + new_version = f'{base_version}-dev{dev_part + 1}' + dependencies[dep] = new_version +data['tool']['poetry']['dependencies'] = dependencies +with open('$pyproject', 'w') as f: + f.write(tomlkit.dumps(data)) +" + done + + - name: Commit changes + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add . + git commit -m "Incremented versions in pyproject.toml files" + + - name: Push changes + uses: ad-m/github-push-action@v0.6.0 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + branch: ${{ github.ref_name }} From 32af6d3d05221ed240f76afd09aec2ac9205b3c3 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 21 Nov 2024 05:52:02 -0600 Subject: [PATCH 024/168] comm - Update pyproject.toml --- pkgs/community/pyproject.toml | 46 +++++++++++++++++------------------ 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/pkgs/community/pyproject.toml b/pkgs/community/pyproject.toml index a2eb33d1c..5f4871f59 100644 --- a/pkgs/community/pyproject.toml +++ b/pkgs/community/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "swarmauri-community" -version = "0.5.2.dev20" +version = "0.5.3.dev2" description = "This repository includes Swarmauri community components." authors = ["Jacob Stewart "] license = "Apache-2.0" @@ -27,9 +27,25 @@ torch = { version = "^2.5.0", optional = true } leptonai = { version = "==0.22.0", optional = true } redis = { version = "^4.0", optional = true } pinecone-client = { version = ">=2.0.0", optional = true, extras = ["grpc"] } -#protobuf = { version = "^3.20.0", optional = true } -#numba = { version = ">=0.59.0", optional = true } -#pacmap = { version = "==0.7.3", optional = true } +textstat = { version = "^0.7.0", optional = true } +annoy = { version = "^1.17.1", optional = true } +folium = { version = "^0.14.0", optional = true } +duckdb = { version = "^0.8.1", optional = true } +neo4j = { version = "^5.12.0", optional = true } +chromadb = { version = "^0.4.4", optional = true } +qdrant_client = { version = "^1.2.0", optional = true } +weaviate-client = { version = "^3.12.0", optional = true } +PyPDF2 = { version = "^3.0.0", optional = true } +pymupdf = { version = "^1.21.1", optional = true } +pypdftk = { version = "^0.4.1", optional = true } +psutil = { version = "^5.9.5", optional = true } +qrcode = { version = "^7.4.0", optional = true } +pygithub = { version = "^1.58.0", optional = true } +gradio = { version = "^3.40.0", optional = true } +google-generativeai = { version = "^0.5.0", optional = true } +openai = { version = "^0.27.0", optional = true } +scipy = { version = "^1.11.0", optional = true } +tiktoken = { version = "^0.3.0", optional = true } [tool.poetry.extras] # Grouped optional dependencies @@ -45,8 +61,6 @@ torch = ["torch"] gradio = ["gradio"] model_clients = ["leptonai", "google-generativeai", "openai"] tiktoken = ["tiktoken"] -#protobuf = ["protobuf"] -#pacmap = ["numba", "pacmap"] # Full installation full = [ @@ -59,8 +73,8 @@ full = [ "scipy", "spacy", "torch", "gradio", - "leptonai", "google-generativeai", "openai" - #"pacmap", "numba" + "leptonai", "google-generativeai", "openai", + "tiktoken" ] [tool.poetry.dev-dependencies] @@ -74,18 +88,4 @@ python-dotenv = "^1.0.0" requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" -[tool.pytest.ini_options] -norecursedirs = ["combined", "scripts"] - -markers = [ - "test: standard test", - "unit: Unit tests", - "integration: Integration tests", - "acceptance: Acceptance tests", - "experimental: Experimental tests" -] -log_cli = true -log_cli_level = "INFO" -log_cli_format = "%(asctime)s [%(levelname)s] %(message)s" -log_cli_date_format = "%Y-%m-%d %H:%M:%S" -asyncio_default_fixture_loop_scope = "function" +[tool.pytest.ini From 31558d1f05e2af64be4526b58ed769335df7c2a9 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 21 Nov 2024 05:52:47 -0600 Subject: [PATCH 025/168] comm - Update pyproject.toml --- pkgs/community/pyproject.toml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/pkgs/community/pyproject.toml b/pkgs/community/pyproject.toml index 5f4871f59..f0ff82388 100644 --- a/pkgs/community/pyproject.toml +++ b/pkgs/community/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "swarmauri-community" -version = "0.5.3.dev2" +version = "0.5.3.dev3" description = "This repository includes Swarmauri community components." authors = ["Jacob Stewart "] license = "Apache-2.0" @@ -88,4 +88,18 @@ python-dotenv = "^1.0.0" requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" -[tool.pytest.ini +[tool.pytest.ini_options] +norecursedirs = ["combined", "scripts"] + +markers = [ + "test: standard test", + "unit: Unit tests", + "integration: Integration tests", + "acceptance: Acceptance tests", + "experimental: Experimental tests" +] +log_cli = true +log_cli_level = "INFO" +log_cli_format = "%(asctime)s [%(levelname)s] %(message)s" +log_cli_date_format = "%Y-%m-%d %H:%M:%S" +asyncio_default_fixture_loop_scope = "function" From 66e42b824a1a901941772d0c62d681233d780e91 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 21 Nov 2024 05:59:28 -0600 Subject: [PATCH 026/168] comm - Update pyproject.toml --- pkgs/community/pyproject.toml | 77 ++++++++++++++++------------------- 1 file changed, 35 insertions(+), 42 deletions(-) diff --git a/pkgs/community/pyproject.toml b/pkgs/community/pyproject.toml index f0ff82388..71dfdb2b0 100644 --- a/pkgs/community/pyproject.toml +++ b/pkgs/community/pyproject.toml @@ -15,65 +15,58 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.13" +captcha = "*" +chromadb = { version = "*", optional = true } +duckdb = { version = "*", optional = true } +folium = { version = "*", optional = true } +gensim = { version = "*", optional = true } +gradio = { version = "*", optional = true } +leptonai = { version = "0.22.0", optional = true } +neo4j = { version = "*", optional = true } +nltk = { version = "*", optional = true } +pandas = "*" +psutil = { version = "*", optional = true } +pygithub = { version = "*", optional = true } +python-dotenv = "*" +qrcode = { version = "*", optional = true } +redis = { version = "^4.0", optional = true } swarmauri = "==0.5.3.dev1" +textstat = { version = "*", optional = true } +transformers = { version = ">=4.45.0", optional = true } typing_extensions = "*" -matplotlib = { version = ">=3.9.2", optional = true } -nltk = { version = "^3.9.1", optional = true } -gensim = { version = "==4.3.3", optional = true } -transformers = { version = "^4.45.0", optional = true } -spacy = { version = ">=3.0.0,<=3.8.2", optional = true } -textblob = { version = "^0.18.0", optional = true } -torch = { version = "^2.5.0", optional = true } -leptonai = { version = "==0.22.0", optional = true } -redis = { version = "^4.0", optional = true } -pinecone-client = { version = ">=2.0.0", optional = true, extras = ["grpc"] } -textstat = { version = "^0.7.0", optional = true } -annoy = { version = "^1.17.1", optional = true } -folium = { version = "^0.14.0", optional = true } -duckdb = { version = "^0.8.1", optional = true } -neo4j = { version = "^5.12.0", optional = true } -chromadb = { version = "^0.4.4", optional = true } -qdrant_client = { version = "^1.2.0", optional = true } -weaviate-client = { version = "^3.12.0", optional = true } -PyPDF2 = { version = "^3.0.0", optional = true } -pymupdf = { version = "^1.21.1", optional = true } -pypdftk = { version = "^0.4.1", optional = true } -psutil = { version = "^5.9.5", optional = true } -qrcode = { version = "^7.4.0", optional = true } -pygithub = { version = "^1.58.0", optional = true } -gradio = { version = "^3.40.0", optional = true } -google-generativeai = { version = "^0.5.0", optional = true } -openai = { version = "^0.27.0", optional = true } -scipy = { version = "^1.11.0", optional = true } -tiktoken = { version = "^0.3.0", optional = true } +tiktoken = { version = "*", optional = true } +pymupdf = { version = "*", optional = true } +annoy = { version = "*", optional = true } +qdrant_client = { version = "*", optional = true } +weaviate = { version = "*", optional = true } +pinecone-client = { version = "*", optional = true, extras = ["grpc"] } +PyPDF2 = { version = "*", optional = true } +pypdftk = { version = "*", optional = true } +weaviate-client = { version = "*", optional = true } +protobuf = { version = "^3.20.0", optional = true } [tool.poetry.extras] # Grouped optional dependencies -nlp = ["nltk", "textblob", "textstat", "gensim"] +nlp = ["nltk", "gensim", "textstat"] ml_toolkits = ["transformers", "annoy"] -visualization = ["folium", "matplotlib"] -storage = ["redis", "duckdb", "neo4j", "chromadb", "qdrant_client", "weaviate-client", "pinecone-client"] +visualization = ["folium"] +storage = ["redis", "duckdb", "neo4j", "chromadb", "qdrant_client", "weaviate", "pinecone-client"] document_processing = ["PyPDF2", "pymupdf", "pypdftk"] cloud_integration = ["psutil", "qrcode", "pygithub"] -spacy = ["spacy"] -transformers = ["transformers"] -torch = ["torch"] gradio = ["gradio"] -model_clients = ["leptonai", "google-generativeai", "openai"] +model_clients = ["leptonai"] tiktoken = ["tiktoken"] # Full installation full = [ "nltk", "gensim", "textstat", "transformers", "annoy", - "folium", "matplotlib", - "redis", "duckdb", "neo4j", "chromadb", "qdrant_client", "weaviate-client", "pinecone-client", + "folium", + "redis", "duckdb", "neo4j", "chromadb", "qdrant_client", "weaviate", "pinecone-client", "PyPDF2", "pymupdf", "pypdftk", "psutil", "qrcode", "pygithub", - "scipy", "spacy", - "torch", "gradio", - "leptonai", "google-generativeai", "openai", + "leptonai", "tiktoken" ] @@ -82,7 +75,7 @@ flake8 = "^7.0" pytest = "^8.0" pytest-asyncio = ">=0.24.0" pytest-xdist = "^3.6.1" -python-dotenv = "^1.0.0" +python-dotenv = "*" [build-system] requires = ["poetry-core>=1.0.0"] From 4ddac14aae565d91f5217bcafe54f2d38fa274a2 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 21 Nov 2024 06:01:53 -0600 Subject: [PATCH 027/168] Update increment_version_dev.yaml --- .github/workflows/increment_version_dev.yaml | 71 ++++++++------------ 1 file changed, 29 insertions(+), 42 deletions(-) diff --git a/.github/workflows/increment_version_dev.yaml b/.github/workflows/increment_version_dev.yaml index 551b83093..2a9736477 100644 --- a/.github/workflows/increment_version_dev.yaml +++ b/.github/workflows/increment_version_dev.yaml @@ -27,49 +27,36 @@ jobs: find . -name "pyproject.toml" | while read -r pyproject; do echo "Processing $pyproject" - # Extract current version - CURRENT_VERSION=$(python -c " -import tomlkit -with open('$pyproject', 'r') as f: - data = tomlkit.parse(f.read()) - print(data['tool']['poetry']['version']) -") + # Extract current version + CURRENT_VERSION=$(python -c " + import tomlkit + try: + with open('$pyproject', 'r') as f: + data = tomlkit.parse(f.read()) + print(data['tool']['poetry']['version']) + except Exception as e: + print('Error reading version:', e) + exit(1) + ") + + if [ -z "$CURRENT_VERSION" ]; then + echo "Error: Could not extract the current version from $pyproject" + exit 1 + fi + + # Increment version + BASE_VERSION=$(echo "$CURRENT_VERSION" | sed -E 's/(.*)-dev.*/\1/') + DEV_PART=$(echo "$CURRENT_VERSION" | grep -oE 'dev[0-9]+$' | grep -oE '[0-9]+') + + if [ -z "$DEV_PART" ]; then + DEV_PART=0 + fi + + NEW_DEV_PART=$((DEV_PART + 1)) + NEW_VERSION="${BASE_VERSION}-dev${NEW_DEV_PART}" + + echo "Updating version from $CURRENT_VERSION to $NEW_VERSION" - # Increment version - BASE_VERSION=$(echo "$CURRENT_VERSION" | sed -E 's/(.*)-dev.*/\1/') - DEV_PART=$(echo "$CURRENT_VERSION" | grep -oE 'dev[0-9]+$' | grep -oE '[0-9]+') - NEW_DEV_PART=$((DEV_PART + 1)) - NEW_VERSION="${BASE_VERSION}-dev${NEW_DEV_PART:-1}" - - echo "Updating version from $CURRENT_VERSION to $NEW_VERSION" - - # Update version in pyproject.toml - python -c " -import tomlkit -with open('$pyproject', 'r') as f: - data = tomlkit.parse(f.read()) -data['tool']['poetry']['version'] = '$NEW_VERSION' -with open('$pyproject', 'w') as f: - f.write(tomlkit.dumps(data)) -" - - # Update dependencies starting with 'swarmauri' - python -c " -import tomlkit -with open('$pyproject', 'r') as f: - data = tomlkit.parse(f.read()) -dependencies = data['tool']['poetry'].get('dependencies', {}) -for dep, version in dependencies.items(): - if dep.startswith('swarmauri') and isinstance(version, str): - base_version = version.split('-dev')[0] - dev_part = int(version.split('-dev')[-1]) if '-dev' in version else 0 - new_version = f'{base_version}-dev{dev_part + 1}' - dependencies[dep] = new_version -data['tool']['poetry']['dependencies'] = dependencies -with open('$pyproject', 'w') as f: - f.write(tomlkit.dumps(data)) -" - done - name: Commit changes run: | From 28c388c590d312300725c1cc26244863a90bfb6e Mon Sep 17 00:00:00 2001 From: michaeldecent2 <111002205+MichaelDecent@users.noreply.github.com> Date: Thu, 21 Nov 2024 14:19:48 +0100 Subject: [PATCH 028/168] comm - Update dependencies in pyproject.toml --- pkgs/community/pyproject.toml | 56 +++++++++++++++++------------------ 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/pkgs/community/pyproject.toml b/pkgs/community/pyproject.toml index 71dfdb2b0..b33cc9ffb 100644 --- a/pkgs/community/pyproject.toml +++ b/pkgs/community/pyproject.toml @@ -15,43 +15,41 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.13" -captcha = "*" -chromadb = { version = "*", optional = true } -duckdb = { version = "*", optional = true } -folium = { version = "*", optional = true } -gensim = { version = "*", optional = true } -gradio = { version = "*", optional = true } -leptonai = { version = "0.22.0", optional = true } -neo4j = { version = "*", optional = true } -nltk = { version = "*", optional = true } -pandas = "*" -psutil = { version = "*", optional = true } -pygithub = { version = "*", optional = true } -python-dotenv = "*" -qrcode = { version = "*", optional = true } +captcha = "^0.6.0" +chromadb = { version = "^0.5.17", optional = true } +duckdb = { version = "^1.1.1", optional = true } +folium = { version = "^0.18.0", optional = true } +gensim = { version = "^4.3.3", optional = true } +gradio = { version = "^5.4.0", optional = true } +leptonai = { version = "^0.22.0", optional = true } +neo4j = { version = "^5.25.0", optional = true } +nltk = { version = "^3.9.1", optional = true } +pandas = "^2.2.3" +psutil = { version = "^6.1.0", optional = true } +pygithub = { version = "^2.4.0", optional = true } +qrcode = { version = "^8.0", optional = true } redis = { version = "^4.0", optional = true } swarmauri = "==0.5.3.dev1" -textstat = { version = "*", optional = true } +textstat = { version = "^0.7.4", optional = true } transformers = { version = ">=4.45.0", optional = true } -typing_extensions = "*" -tiktoken = { version = "*", optional = true } -pymupdf = { version = "*", optional = true } -annoy = { version = "*", optional = true } -qdrant_client = { version = "*", optional = true } -weaviate = { version = "*", optional = true } -pinecone-client = { version = "*", optional = true, extras = ["grpc"] } -PyPDF2 = { version = "*", optional = true } -pypdftk = { version = "*", optional = true } -weaviate-client = { version = "*", optional = true } -protobuf = { version = "^3.20.0", optional = true } +typing_extensions = "^4.12.2" +tiktoken = { version = "^0.8.0", optional = true } +PyMuPDF = { version = "^1.24.12", optional = true } +annoy = { version = "^1.17.3", optional = true } +qdrant-client = { version = "^1.12.0", optional = true } +pinecone-client = { version = "^5.0.1", optional = true, extras = ["grpc"] } +pypdf = { version = "^5.0.1", optional = true } +pypdftk = { version = "^0.5", optional = true } +weaviate-client = { version = "^4.9.2", optional = true } +#protobuf = { version = "^3.20.0", optional = true } [tool.poetry.extras] # Grouped optional dependencies nlp = ["nltk", "gensim", "textstat"] ml_toolkits = ["transformers", "annoy"] visualization = ["folium"] -storage = ["redis", "duckdb", "neo4j", "chromadb", "qdrant_client", "weaviate", "pinecone-client"] -document_processing = ["PyPDF2", "pymupdf", "pypdftk"] +storage = ["redis", "duckdb", "neo4j", "chromadb", "qdrant-client", "weaviate", "pinecone-client"] +document_processing = ["pypdf", "PyMuPDF", "pypdftk"] cloud_integration = ["psutil", "qrcode", "pygithub"] gradio = ["gradio"] model_clients = ["leptonai"] @@ -63,7 +61,7 @@ full = [ "transformers", "annoy", "folium", "redis", "duckdb", "neo4j", "chromadb", "qdrant_client", "weaviate", "pinecone-client", - "PyPDF2", "pymupdf", "pypdftk", + "pypdf", "PyMuPDF", "pypdftk", "psutil", "qrcode", "pygithub", "gradio", "leptonai", From fb2bb4f889f5a2a893b887062151f02124ddcfc9 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 21 Nov 2024 07:48:24 -0600 Subject: [PATCH 029/168] swarm - Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index 27656153a..ed5b5e509 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -15,7 +15,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.13" -swarmauri_core = "==0.5.2" +swarmauri_core = "==0.5.3.dev2" toml = "^0.10.2" httpx = "^0.27.2" joblib = "^1.4.0" From 940b74ffa923c18a35f96f19e53a749d3f121914 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 21 Nov 2024 07:49:14 -0600 Subject: [PATCH 030/168] core - Update pyproject.toml --- pkgs/core/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/core/pyproject.toml b/pkgs/core/pyproject.toml index 4f5283c21..dd96deb27 100644 --- a/pkgs/core/pyproject.toml +++ b/pkgs/core/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "swarmauri-core" -version = "0.5.2" +version = "0.5.3.dev2" description = "This repository includes core interfaces for the Swarmauri framework." authors = ["Jacob Stewart "] license = "Apache-2.0" From 7c486db7a55fc4cec65a32dd8160d737cb4c5459 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 21 Nov 2024 07:49:55 -0600 Subject: [PATCH 031/168] core - Update pyproject.toml --- pkgs/core/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/core/pyproject.toml b/pkgs/core/pyproject.toml index dd96deb27..887619499 100644 --- a/pkgs/core/pyproject.toml +++ b/pkgs/core/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "swarmauri-core" -version = "0.5.3.dev2" +version = "0.5.3.dev3" description = "This repository includes core interfaces for the Swarmauri framework." authors = ["Jacob Stewart "] license = "Apache-2.0" From 4a6d9d6ced8659ba1360be79b8ffc48af496eecc Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 21 Nov 2024 07:50:05 -0600 Subject: [PATCH 032/168] swarm - Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index ed5b5e509..4db045b85 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "swarmauri" -version = "0.5.3.dev2" +version = "0.5.3.dev3" description = "This repository includes base classes, concrete generics, and concrete standard components within the Swarmauri framework." authors = ["Jacob Stewart "] license = "Apache-2.0" @@ -15,7 +15,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.13" -swarmauri_core = "==0.5.3.dev2" +swarmauri_core = "==0.5.3.dev3" toml = "^0.10.2" httpx = "^0.27.2" joblib = "^1.4.0" From a1e0967e5061f931872e6064c417d7c28fdd76f7 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 21 Nov 2024 07:50:21 -0600 Subject: [PATCH 033/168] comm - Update pyproject.toml --- pkgs/community/pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/community/pyproject.toml b/pkgs/community/pyproject.toml index b33cc9ffb..e500a0d0f 100644 --- a/pkgs/community/pyproject.toml +++ b/pkgs/community/pyproject.toml @@ -4,7 +4,7 @@ version = "0.5.3.dev3" description = "This repository includes Swarmauri community components." authors = ["Jacob Stewart "] license = "Apache-2.0" -readme = "README.md" +readme = "README.md"e repository = "http://github.com/swarmauri/swarmauri-sdk" classifiers = [ "License :: OSI Approved :: Apache Software License", @@ -29,7 +29,7 @@ psutil = { version = "^6.1.0", optional = true } pygithub = { version = "^2.4.0", optional = true } qrcode = { version = "^8.0", optional = true } redis = { version = "^4.0", optional = true } -swarmauri = "==0.5.3.dev1" +swarmauri = "==0.5.3.dev3" textstat = { version = "^0.7.4", optional = true } transformers = { version = ">=4.45.0", optional = true } typing_extensions = "^4.12.2" From b015b32aa724e8d63be427fbb27f1168acef206c Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 21 Nov 2024 07:50:39 -0600 Subject: [PATCH 034/168] Update pyproject.toml --- pkgs/experimental/pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/experimental/pyproject.toml b/pkgs/experimental/pyproject.toml index bea605df3..1d359131a 100644 --- a/pkgs/experimental/pyproject.toml +++ b/pkgs/experimental/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "swarmauri-experimental" -version = "0.5.2" +version = "0.5.3.dev3" description = "This repository includes experimental components." authors = ["Jacob Stewart "] license = "Apache-2.0" @@ -15,7 +15,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<4.0" -swarmauri = "==0.5.2" +swarmauri = "==0.5.3.dev3" gensim = "*" neo4j = "*" numpy = "*" From b57270f5416c755ffbbb993ba06b0ff07795f995 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 21 Nov 2024 07:53:55 -0600 Subject: [PATCH 035/168] swarm - Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index 4db045b85..26317b09f 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "swarmauri" -version = "0.5.3.dev3" +version = "0.5.3.dev4" description = "This repository includes base classes, concrete generics, and concrete standard components within the Swarmauri framework." authors = ["Jacob Stewart "] license = "Apache-2.0" @@ -17,7 +17,7 @@ classifiers = [ python = ">=3.10,<3.13" swarmauri_core = "==0.5.3.dev3" toml = "^0.10.2" -httpx = "^0.27.2" +httpx = "^0.25.0" joblib = "^1.4.0" numpy = "*" pandas = "*" From 9aaa12d1d75690fb17488e5fabcdf75fd38f9151 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 21 Nov 2024 07:54:13 -0600 Subject: [PATCH 036/168] comm - Update pyproject.toml --- pkgs/community/pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/community/pyproject.toml b/pkgs/community/pyproject.toml index e500a0d0f..86112c335 100644 --- a/pkgs/community/pyproject.toml +++ b/pkgs/community/pyproject.toml @@ -1,10 +1,10 @@ [tool.poetry] name = "swarmauri-community" -version = "0.5.3.dev3" +version = "0.5.3.dev4" description = "This repository includes Swarmauri community components." authors = ["Jacob Stewart "] license = "Apache-2.0" -readme = "README.md"e +readme = "README.md" repository = "http://github.com/swarmauri/swarmauri-sdk" classifiers = [ "License :: OSI Approved :: Apache Software License", @@ -29,7 +29,7 @@ psutil = { version = "^6.1.0", optional = true } pygithub = { version = "^2.4.0", optional = true } qrcode = { version = "^8.0", optional = true } redis = { version = "^4.0", optional = true } -swarmauri = "==0.5.3.dev3" +swarmauri = "==0.5.3.dev4" textstat = { version = "^0.7.4", optional = true } transformers = { version = ">=4.45.0", optional = true } typing_extensions = "^4.12.2" From 6820a6f180b1036e6c6853615a64455bc3e24dc6 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 21 Nov 2024 08:07:13 -0600 Subject: [PATCH 037/168] Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index 26317b09f..14878d8e4 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -17,7 +17,7 @@ classifiers = [ python = ">=3.10,<3.13" swarmauri_core = "==0.5.3.dev3" toml = "^0.10.2" -httpx = "^0.25.0" +httpx = "^0.27.0" joblib = "^1.4.0" numpy = "*" pandas = "*" From c70f09f55f5b9406cf33bbe9373bc3ff5c17a72b Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 21 Nov 2024 08:07:28 -0600 Subject: [PATCH 038/168] swarm - Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index 14878d8e4..23ad0e37f 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "swarmauri" -version = "0.5.3.dev4" +version = "0.5.3.dev5" description = "This repository includes base classes, concrete generics, and concrete standard components within the Swarmauri framework." authors = ["Jacob Stewart "] license = "Apache-2.0" From 1257f30add771bf74e46a1ce28714f9e6880d378 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 21 Nov 2024 08:07:46 -0600 Subject: [PATCH 039/168] swarm - Update pyproject.toml --- pkgs/community/pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/community/pyproject.toml b/pkgs/community/pyproject.toml index 86112c335..256adb6da 100644 --- a/pkgs/community/pyproject.toml +++ b/pkgs/community/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "swarmauri-community" -version = "0.5.3.dev4" +version = "0.5.3.dev5" description = "This repository includes Swarmauri community components." authors = ["Jacob Stewart "] license = "Apache-2.0" @@ -29,7 +29,7 @@ psutil = { version = "^6.1.0", optional = true } pygithub = { version = "^2.4.0", optional = true } qrcode = { version = "^8.0", optional = true } redis = { version = "^4.0", optional = true } -swarmauri = "==0.5.3.dev4" +swarmauri = "==0.5.3.dev5" textstat = { version = "^0.7.4", optional = true } transformers = { version = ">=4.45.0", optional = true } typing_extensions = "^4.12.2" From d7bc92ad7cddc85542e2306a61275fbae1b4108f Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 21 Nov 2024 08:08:05 -0600 Subject: [PATCH 040/168] exp - Update pyproject.toml --- pkgs/experimental/pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/experimental/pyproject.toml b/pkgs/experimental/pyproject.toml index 1d359131a..78339dbbc 100644 --- a/pkgs/experimental/pyproject.toml +++ b/pkgs/experimental/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "swarmauri-experimental" -version = "0.5.3.dev3" +version = "0.5.3.dev5" description = "This repository includes experimental components." authors = ["Jacob Stewart "] license = "Apache-2.0" @@ -15,7 +15,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<4.0" -swarmauri = "==0.5.3.dev3" +swarmauri = "==0.5.3.dev5" gensim = "*" neo4j = "*" numpy = "*" From d44a15dbfbbc430f8e23c38457f19b3c26adf589 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 21 Nov 2024 08:13:36 -0600 Subject: [PATCH 041/168] cicd - Update sequence_publish.yaml --- .github/workflows/sequence_publish.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sequence_publish.yaml b/.github/workflows/sequence_publish.yaml index 23a88d990..f219023f4 100644 --- a/.github/workflows/sequence_publish.yaml +++ b/.github/workflows/sequence_publish.yaml @@ -93,7 +93,7 @@ jobs: - uses: actions/checkout@v4 - name: Wait for swarmauri - run: sleep 60 + run: sleep 120 - name: Set up Python 3.12 uses: actions/setup-python@v5 From 51c1a44fc6342d374c9b657f18844a6b99a471a7 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 21 Nov 2024 08:44:44 -0600 Subject: [PATCH 042/168] exp - [Update pyproject.toml --- pkgs/experimental/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/experimental/pyproject.toml b/pkgs/experimental/pyproject.toml index 78339dbbc..2bcdb3b7f 100644 --- a/pkgs/experimental/pyproject.toml +++ b/pkgs/experimental/pyproject.toml @@ -14,7 +14,7 @@ classifiers = [ ] [tool.poetry.dependencies] -python = ">=3.10,<4.0" +python = ">=3.10,<3.13" swarmauri = "==0.5.3.dev5" gensim = "*" neo4j = "*" From 99784edda7be07ef462a72a0e12149d931b15015 Mon Sep 17 00:00:00 2001 From: michaeldecent2 <111002205+MichaelDecent@users.noreply.github.com> Date: Thu, 21 Nov 2024 16:37:50 +0100 Subject: [PATCH 043/168] comm - fix errors related to community --- pkgs/community/pyproject.toml | 10 ++++++---- .../vector_stores/concrete/AnnoyVectorStore.py | 2 +- .../vector_stores/concrete/CloudQdrantVectorStore.py | 2 +- .../vector_stores/concrete/CloudWeaviateVectorStore.py | 2 +- .../vector_stores/concrete/DuckDBVectorStore.py | 2 +- .../vector_stores/concrete/MlmVectorStore.py | 3 ++- .../vector_stores/concrete/Neo4jVectorStore.py | 2 +- .../concrete/PersistentChromaDBVectorStore.py | 2 +- .../concrete/PersistentQdrantVectorStore.py | 2 +- .../vector_stores/concrete/PineconeVectorStore.py | 2 +- .../vector_stores/concrete/RedisVectorStore.py | 7 +++---- 11 files changed, 19 insertions(+), 17 deletions(-) diff --git a/pkgs/community/pyproject.toml b/pkgs/community/pyproject.toml index 256adb6da..90f870f2b 100644 --- a/pkgs/community/pyproject.toml +++ b/pkgs/community/pyproject.toml @@ -41,14 +41,16 @@ pinecone-client = { version = "^5.0.1", optional = true, extras = ["grpc"] } pypdf = { version = "^5.0.1", optional = true } pypdftk = { version = "^0.5", optional = true } weaviate-client = { version = "^4.9.2", optional = true } +textblob = { version = "^0.18.0", optional = true } +torch = { version = "^2.4.1", optional = true} #protobuf = { version = "^3.20.0", optional = true } [tool.poetry.extras] # Grouped optional dependencies -nlp = ["nltk", "gensim", "textstat"] +nlp = ["nltk", "gensim", "textstat", "textblob", "torch"] ml_toolkits = ["transformers", "annoy"] visualization = ["folium"] -storage = ["redis", "duckdb", "neo4j", "chromadb", "qdrant-client", "weaviate", "pinecone-client"] +storage = ["redis", "duckdb", "neo4j", "chromadb", "qdrant-client", "weaviate-client", "pinecone-client"] document_processing = ["pypdf", "PyMuPDF", "pypdftk"] cloud_integration = ["psutil", "qrcode", "pygithub"] gradio = ["gradio"] @@ -57,10 +59,10 @@ tiktoken = ["tiktoken"] # Full installation full = [ - "nltk", "gensim", "textstat", + "nltk", "gensim", "textstat", "textblob", "torch", "transformers", "annoy", "folium", - "redis", "duckdb", "neo4j", "chromadb", "qdrant_client", "weaviate", "pinecone-client", + "redis", "duckdb", "neo4j", "chromadb", "qdrant-client", "weaviate-client", "pinecone-client", "pypdf", "PyMuPDF", "pypdftk", "psutil", "qrcode", "pygithub", "gradio", diff --git a/pkgs/community/swarmauri_community/vector_stores/concrete/AnnoyVectorStore.py b/pkgs/community/swarmauri_community/vector_stores/concrete/AnnoyVectorStore.py index 354b2c2fe..bcc4fc214 100644 --- a/pkgs/community/swarmauri_community/vector_stores/concrete/AnnoyVectorStore.py +++ b/pkgs/community/swarmauri_community/vector_stores/concrete/AnnoyVectorStore.py @@ -4,7 +4,7 @@ import os from swarmauri.documents.concrete.Document import Document -from swarmauri.embeddings.concrete.Doc2VecEmbedding import Doc2VecEmbedding +from swarmauri_community.embeddings.concrete.Doc2VecEmbedding import Doc2VecEmbedding from swarmauri.distances.concrete.CosineDistance import CosineDistance from swarmauri.vector_stores.base.VectorStoreBase import VectorStoreBase diff --git a/pkgs/community/swarmauri_community/vector_stores/concrete/CloudQdrantVectorStore.py b/pkgs/community/swarmauri_community/vector_stores/concrete/CloudQdrantVectorStore.py index e1af20b9a..28f82d250 100644 --- a/pkgs/community/swarmauri_community/vector_stores/concrete/CloudQdrantVectorStore.py +++ b/pkgs/community/swarmauri_community/vector_stores/concrete/CloudQdrantVectorStore.py @@ -10,7 +10,7 @@ ) from swarmauri.documents.concrete.Document import Document -from swarmauri.embeddings.concrete.Doc2VecEmbedding import Doc2VecEmbedding +from swarmauri_community.embeddings.concrete.Doc2VecEmbedding import Doc2VecEmbedding from swarmauri.distances.concrete.CosineDistance import CosineDistance from swarmauri.vector_stores.base.VectorStoreBase import VectorStoreBase diff --git a/pkgs/community/swarmauri_community/vector_stores/concrete/CloudWeaviateVectorStore.py b/pkgs/community/swarmauri_community/vector_stores/concrete/CloudWeaviateVectorStore.py index 0e1ffced7..770528174 100644 --- a/pkgs/community/swarmauri_community/vector_stores/concrete/CloudWeaviateVectorStore.py +++ b/pkgs/community/swarmauri_community/vector_stores/concrete/CloudWeaviateVectorStore.py @@ -7,7 +7,7 @@ from weaviate.classes.query import MetadataQuery from swarmauri.documents.concrete.Document import Document -from swarmauri.embeddings.concrete.Doc2VecEmbedding import Doc2VecEmbedding +from swarmauri_community.embeddings.concrete.Doc2VecEmbedding import Doc2VecEmbedding from swarmauri.vectors.concrete.Vector import Vector from swarmauri.vector_stores.base.VectorStoreBase import VectorStoreBase diff --git a/pkgs/community/swarmauri_community/vector_stores/concrete/DuckDBVectorStore.py b/pkgs/community/swarmauri_community/vector_stores/concrete/DuckDBVectorStore.py index 03d3baf7a..c415c5b6a 100644 --- a/pkgs/community/swarmauri_community/vector_stores/concrete/DuckDBVectorStore.py +++ b/pkgs/community/swarmauri_community/vector_stores/concrete/DuckDBVectorStore.py @@ -7,7 +7,7 @@ from swarmauri.vectors.concrete.Vector import Vector from swarmauri.documents.concrete.Document import Document -from swarmauri.embeddings.concrete.Doc2VecEmbedding import Doc2VecEmbedding +from swarmauri_community.embeddings.concrete.Doc2VecEmbedding import Doc2VecEmbedding from swarmauri.distances.concrete.CosineDistance import CosineDistance from swarmauri.vector_stores.base.VectorStoreBase import VectorStoreBase diff --git a/pkgs/community/swarmauri_community/vector_stores/concrete/MlmVectorStore.py b/pkgs/community/swarmauri_community/vector_stores/concrete/MlmVectorStore.py index 21f45ec35..1fd98eed9 100644 --- a/pkgs/community/swarmauri_community/vector_stores/concrete/MlmVectorStore.py +++ b/pkgs/community/swarmauri_community/vector_stores/concrete/MlmVectorStore.py @@ -5,7 +5,8 @@ from swarmauri.vector_stores.base.VectorStoreBase import VectorStoreBase from swarmauri.vector_stores.base.VectorStoreRetrieveMixin import VectorStoreRetrieveMixin -from swarmauri.vector_stores.base.VectorStoreSaveLoadMixin import VectorStoreSaveLoadMixin +from swarmauri.vector_stores.base.VectorStoreSaveLoadMixin import VectorStoreSaveLoadMixin + class MlmVectorStore(VectorStoreSaveLoadMixin, VectorStoreRetrieveMixin, VectorStoreBase): type: Literal['MlmVectorStore'] = 'MlmVectorStore' diff --git a/pkgs/community/swarmauri_community/vector_stores/concrete/Neo4jVectorStore.py b/pkgs/community/swarmauri_community/vector_stores/concrete/Neo4jVectorStore.py index 75f162283..a67b1da37 100644 --- a/pkgs/community/swarmauri_community/vector_stores/concrete/Neo4jVectorStore.py +++ b/pkgs/community/swarmauri_community/vector_stores/concrete/Neo4jVectorStore.py @@ -1,5 +1,5 @@ from typing import List, Union, Literal, Optional -from pydantic import BaseModel, PrivateAttr, field_validator +from pydantic import BaseModel, PrivateAttr from neo4j import GraphDatabase import json diff --git a/pkgs/community/swarmauri_community/vector_stores/concrete/PersistentChromaDBVectorStore.py b/pkgs/community/swarmauri_community/vector_stores/concrete/PersistentChromaDBVectorStore.py index 83e16a7ae..413a89c25 100644 --- a/pkgs/community/swarmauri_community/vector_stores/concrete/PersistentChromaDBVectorStore.py +++ b/pkgs/community/swarmauri_community/vector_stores/concrete/PersistentChromaDBVectorStore.py @@ -5,7 +5,7 @@ from typing import List, Union, Literal from swarmauri.documents.concrete.Document import Document -from swarmauri.embeddings.concrete.Doc2VecEmbedding import Doc2VecEmbedding +from swarmauri_community.embeddings.concrete.Doc2VecEmbedding import Doc2VecEmbedding from swarmauri.distances.concrete.CosineDistance import CosineDistance from swarmauri.vector_stores.base.VectorStoreBase import VectorStoreBase diff --git a/pkgs/community/swarmauri_community/vector_stores/concrete/PersistentQdrantVectorStore.py b/pkgs/community/swarmauri_community/vector_stores/concrete/PersistentQdrantVectorStore.py index 2aab53309..e22b7c802 100644 --- a/pkgs/community/swarmauri_community/vector_stores/concrete/PersistentQdrantVectorStore.py +++ b/pkgs/community/swarmauri_community/vector_stores/concrete/PersistentQdrantVectorStore.py @@ -9,7 +9,7 @@ ) from swarmauri.documents.concrete.Document import Document -from swarmauri.embeddings.concrete.Doc2VecEmbedding import Doc2VecEmbedding +from swarmauri_community.embeddings.concrete.Doc2VecEmbedding import Doc2VecEmbedding from swarmauri.distances.concrete.CosineDistance import CosineDistance from swarmauri.vector_stores.base.VectorStoreBase import VectorStoreBase diff --git a/pkgs/community/swarmauri_community/vector_stores/concrete/PineconeVectorStore.py b/pkgs/community/swarmauri_community/vector_stores/concrete/PineconeVectorStore.py index a10421a78..d5817aee2 100644 --- a/pkgs/community/swarmauri_community/vector_stores/concrete/PineconeVectorStore.py +++ b/pkgs/community/swarmauri_community/vector_stores/concrete/PineconeVectorStore.py @@ -5,7 +5,7 @@ from pinecone import ServerlessSpec from swarmauri.documents.concrete.Document import Document -from swarmauri.embeddings.concrete.Doc2VecEmbedding import Doc2VecEmbedding +from swarmauri_community.embeddings.concrete.Doc2VecEmbedding import Doc2VecEmbedding from swarmauri.distances.concrete.CosineDistance import CosineDistance from swarmauri.vector_stores.base.VectorStoreBase import VectorStoreBase diff --git a/pkgs/community/swarmauri_community/vector_stores/concrete/RedisVectorStore.py b/pkgs/community/swarmauri_community/vector_stores/concrete/RedisVectorStore.py index 2b77026ac..8264de90f 100644 --- a/pkgs/community/swarmauri_community/vector_stores/concrete/RedisVectorStore.py +++ b/pkgs/community/swarmauri_community/vector_stores/concrete/RedisVectorStore.py @@ -1,16 +1,15 @@ import json -from typing import List, Union, Literal, Dict, Optional +from typing import List, Union, Literal, Optional from pydantic import PrivateAttr import numpy as np import redis -from redis.commands.search.field import VectorField, TextField, TagField +from redis.commands.search.field import VectorField, TextField from redis.commands.search.indexDefinition import IndexDefinition, IndexType -from redis.commands.search.query import Query from swarmauri.vectors.concrete.Vector import Vector from swarmauri.documents.concrete.Document import Document -from swarmauri.embeddings.concrete.Doc2VecEmbedding import Doc2VecEmbedding # or your specific embedder +from swarmauri_community.embeddings.concrete.Doc2VecEmbedding import Doc2VecEmbedding # or your specific embedder from swarmauri.vector_stores.base.VectorStoreBase import VectorStoreBase from swarmauri.vector_stores.base.VectorStoreRetrieveMixin import VectorStoreRetrieveMixin from swarmauri.vector_stores.base.VectorStoreSaveLoadMixin import VectorStoreSaveLoadMixin From e1b93762197ce9097c5a7a5a1971476300ffe9dc Mon Sep 17 00:00:00 2001 From: michaeldecent2 <111002205+MichaelDecent@users.noreply.github.com> Date: Tue, 26 Nov 2024 08:06:54 +0100 Subject: [PATCH 044/168] comm - Add scikit-learn as an optional dependency in pyproject.toml --- pkgs/community/pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkgs/community/pyproject.toml b/pkgs/community/pyproject.toml index 90f870f2b..db711b327 100644 --- a/pkgs/community/pyproject.toml +++ b/pkgs/community/pyproject.toml @@ -43,11 +43,12 @@ pypdftk = { version = "^0.5", optional = true } weaviate-client = { version = "^4.9.2", optional = true } textblob = { version = "^0.18.0", optional = true } torch = { version = "^2.4.1", optional = true} +scikit-learn = { version = "^1.5.2", optional = true } #protobuf = { version = "^3.20.0", optional = true } [tool.poetry.extras] # Grouped optional dependencies -nlp = ["nltk", "gensim", "textstat", "textblob", "torch"] +nlp = ["nltk", "gensim", "textstat", "textblob", "torch", "scikit-learn"] ml_toolkits = ["transformers", "annoy"] visualization = ["folium"] storage = ["redis", "duckdb", "neo4j", "chromadb", "qdrant-client", "weaviate-client", "pinecone-client"] @@ -59,7 +60,7 @@ tiktoken = ["tiktoken"] # Full installation full = [ - "nltk", "gensim", "textstat", "textblob", "torch", + "nltk", "gensim", "textstat", "textblob", "torch", "scikit-learn", "transformers", "annoy", "folium", "redis", "duckdb", "neo4j", "chromadb", "qdrant-client", "weaviate-client", "pinecone-client", From 34619083c1ec68d3f87a0977b504bd8f2cad3b0d Mon Sep 17 00:00:00 2001 From: 3rdSon Date: Wed, 27 Nov 2024 13:26:22 +0100 Subject: [PATCH 045/168] swarm - added dataconnector --- pkgs/core/swarmauri_core/ComponentBase.py | 1 + .../dataconnectors/IDataConnector.py | 70 ++++++++++++++++ .../swarmauri_core/dataconnectors/__init__.py | 0 .../dataconnectors/base/DataConnectorBase.py | 79 +++++++++++++++++++ .../swarmauri/dataconnectors/base/__init__.py | 0 5 files changed, 150 insertions(+) create mode 100644 pkgs/core/swarmauri_core/dataconnectors/IDataConnector.py create mode 100644 pkgs/core/swarmauri_core/dataconnectors/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/dataconnectors/base/DataConnectorBase.py create mode 100644 pkgs/swarmauri/swarmauri/dataconnectors/base/__init__.py diff --git a/pkgs/core/swarmauri_core/ComponentBase.py b/pkgs/core/swarmauri_core/ComponentBase.py index 4336b2047..13a096001 100644 --- a/pkgs/core/swarmauri_core/ComponentBase.py +++ b/pkgs/core/swarmauri_core/ComponentBase.py @@ -52,6 +52,7 @@ class ResourceTypes(Enum): VECTOR_STORE = "VectorStore" VECTOR = "Vector" VCM = "VCM" + DATA_CONNECTOR = "DataConnector" def generate_id() -> str: diff --git a/pkgs/core/swarmauri_core/dataconnectors/IDataConnector.py b/pkgs/core/swarmauri_core/dataconnectors/IDataConnector.py new file mode 100644 index 000000000..b0c7826c4 --- /dev/null +++ b/pkgs/core/swarmauri_core/dataconnectors/IDataConnector.py @@ -0,0 +1,70 @@ +from abc import ABC, abstractmethod + + +class IDataConnector(ABC): + """ + Abstract base class for data connectors. + Defines the interface for all concrete data connector implementations. + """ + + @abstractmethod + def authenticate(self, **kwargs): + """ + Authenticate with the data source. + This method should handle any required authentication process. + + :param kwargs: Authentication parameters such as API keys, tokens, etc. + """ + pass + + @abstractmethod + def fetch_data(self, query: str, **kwargs): + """ + Fetch data from the data source based on a query. + + :param query: Query string or parameters to fetch specific data. + :param kwargs: Additional parameters for fetching data. + :return: Data fetched from the source. + """ + pass + + @abstractmethod + def insert_data(self, data, **kwargs): + """ + Insert data into the data source. + + :param data: Data to be inserted. + :param kwargs: Additional parameters for inserting data. + """ + pass + + @abstractmethod + def update_data(self, identifier, data, **kwargs): + """ + Update existing data in the data source. + + :param identifier: Unique identifier of the data to update. + :param data: Updated data. + :param kwargs: Additional parameters for updating data. + """ + pass + + @abstractmethod + def delete_data(self, identifier, **kwargs): + """ + Delete data from the data source. + + :param identifier: Unique identifier of the data to delete. + :param kwargs: Additional parameters for deleting data. + """ + pass + + @abstractmethod + def test_connection(self, **kwargs): + """ + Test the connection to the data source. + + :param kwargs: Connection parameters. + :return: Boolean indicating whether the connection is successful. + """ + pass diff --git a/pkgs/core/swarmauri_core/dataconnectors/__init__.py b/pkgs/core/swarmauri_core/dataconnectors/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/dataconnectors/base/DataConnectorBase.py b/pkgs/swarmauri/swarmauri/dataconnectors/base/DataConnectorBase.py new file mode 100644 index 000000000..0cb70a202 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/dataconnectors/base/DataConnectorBase.py @@ -0,0 +1,79 @@ +from swarmauri_core.dataconnectors.IDataConnector import IDataConnector + + +class DataConnectorBase(IDataConnector): + """ + Base implementation of IDataConnector that raises NotImplementedError + for all abstract methods, ensuring explicit implementation in child classes. + """ + + def authenticate(self, **kwargs): + """ + Raises NotImplementedError to enforce implementation in child classes. + + :param kwargs: Authentication parameters + :raises NotImplementedError: Always raised to require specific implementation + """ + raise NotImplementedError( + "Authenticate method must be implemented by child classes." + ) + + def fetch_data(self, query: str, **kwargs): + """ + Raises NotImplementedError to enforce implementation in child classes. + + :param query: Query string or parameters + :param kwargs: Additional parameters + :raises NotImplementedError: Always raised to require specific implementation + """ + raise NotImplementedError( + "Fetch data method must be implemented by child classes." + ) + + def insert_data(self, data, **kwargs): + """ + Raises NotImplementedError to enforce implementation in child classes. + + :param data: Data to be inserted + :param kwargs: Additional parameters + :raises NotImplementedError: Always raised to require specific implementation + """ + raise NotImplementedError( + "Insert data method must be implemented by child classes." + ) + + def update_data(self, identifier, data, **kwargs): + """ + Raises NotImplementedError to enforce implementation in child classes. + + :param identifier: Unique identifier of the data to update + :param data: Updated data + :param kwargs: Additional parameters + :raises NotImplementedError: Always raised to require specific implementation + """ + raise NotImplementedError( + "Update data method must be implemented by child classes." + ) + + def delete_data(self, identifier, **kwargs): + """ + Raises NotImplementedError to enforce implementation in child classes. + + :param identifier: Unique identifier of the data to delete + :param kwargs: Additional parameters + :raises NotImplementedError: Always raised to require specific implementation + """ + raise NotImplementedError( + "Delete data method must be implemented by child classes." + ) + + def test_connection(self, **kwargs): + """ + Raises NotImplementedError to enforce implementation in child classes. + + :param kwargs: Connection parameters + :raises NotImplementedError: Always raised to require specific implementation + """ + raise NotImplementedError( + "Test connection method must be implemented by child classes." + ) diff --git a/pkgs/swarmauri/swarmauri/dataconnectors/base/__init__.py b/pkgs/swarmauri/swarmauri/dataconnectors/base/__init__.py new file mode 100644 index 000000000..e69de29bb From 380d0f57e02767c39e857d9da291bbb7dd41accb Mon Sep 17 00:00:00 2001 From: 3rdSon Date: Sat, 30 Nov 2024 11:11:07 +0100 Subject: [PATCH 046/168] swarm -googledataconnector --- .../concrete/GoogleDriveDataConnector.py | 341 ++++++++++++++++++ .../dataconnectors/concrete/__init__.py | 16 + pkgs/swarmauri/tests/static/credentials.json | 5 + .../GoogleDriveDataConnector_unit_test.py | 86 +++++ 4 files changed, 448 insertions(+) create mode 100644 pkgs/swarmauri/swarmauri/dataconnectors/concrete/GoogleDriveDataConnector.py create mode 100644 pkgs/swarmauri/swarmauri/dataconnectors/concrete/__init__.py create mode 100644 pkgs/swarmauri/tests/static/credentials.json create mode 100644 pkgs/swarmauri/tests/unit/dataconnectors/GoogleDriveDataConnector_unit_test.py diff --git a/pkgs/swarmauri/swarmauri/dataconnectors/concrete/GoogleDriveDataConnector.py b/pkgs/swarmauri/swarmauri/dataconnectors/concrete/GoogleDriveDataConnector.py new file mode 100644 index 000000000..57fa66a2a --- /dev/null +++ b/pkgs/swarmauri/swarmauri/dataconnectors/concrete/GoogleDriveDataConnector.py @@ -0,0 +1,341 @@ +import logging +from urllib.parse import urlencode +import httpx +import base64 +import json +from typing import List +from swarmauri.documents.base.DocumentBase import DocumentBase +from swarmauri.dataconnectors.base.DataConnectorBase import DataConnectorBase + + +class GoogleDriveDataConnector(DataConnectorBase): + """ + Data connector for interacting with Google Drive files and converting them to Swarmauri documents. + + Supports authentication, data fetching, and basic CRUD operations for Google Drive resources. + """ + + def __init__(self, credentials_path: str = None): + """ + Initialize the Google Drive Data Connector. + + :param credentials_path: Path to the Google OAuth2 credentials JSON file + """ + with open(credentials_path, "r") as cred_file: + credentials = json.load(cred_file) + + self.client_id = credentials.get("client_id") + self.client_secret = credentials.get("client_secret") + self.redirect_uri = credentials.get("redirect_uri") + + # Tokens will be stored here + self.access_token = None + self.refresh_token = None + + self.authorization_code = None + + self.client = httpx.Client() + + def generate_authorization_url(self) -> str: + """Generate the authorization URL for user consent""" + params = { + "client_id": self.client_id, + "redirect_uri": self.redirect_uri, + "response_type": "code", + "scope": "https://www.googleapis.com/auth/drive", + "access_type": "offline", # This ensures we get a refresh token + } + return f"https://accounts.google.com/o/oauth2/v2/auth?{urlencode(params)}" + + def _exchange_code_for_tokens(self): + """Exchange authorization code for access and refresh tokens""" + if not self.authorization_code: + raise ValueError("No authorization code available") + + token_url = "https://oauth2.googleapis.com/token" + payload = { + "client_id": self.client_id, + "client_secret": self.client_secret, + "code": self.authorization_code, + "grant_type": "authorization_code", + "redirect_uri": self.redirect_uri, + } + + response = self.client.post(token_url, data=payload) + tokens = response.json() + + logging.info(f"Token response: {tokens}") + if "access_token" not in tokens: + raise ValueError("Failed to obtain access token") + self.access_token = tokens["access_token"] + self.refresh_token = tokens.get("refresh_token") + + def refresh_access_token(self): + """Refresh the access token using the refresh token""" + if not self.refresh_token: + raise ValueError("No refresh token available") + + token_url = "https://oauth2.googleapis.com/token" + payload = { + "client_id": self.client_id, + "client_secret": self.client_secret, + "refresh_token": self.refresh_token, + "grant_type": "refresh_token", + } + + response = self.client.post(token_url, data=payload) + tokens = response.json() + self.access_token = tokens["access_token"] + + def authenticate(self): + """ + Authenticate with Google Drive using OAuth2. + + This method generates an authorization URL, prompts the user to visit the URL + and enter the authorization code, and then exchanges the code for tokens. + """ + try: + # Generate authorization URL + auth_url = self.generate_authorization_url() + print("Please visit the following URL to authenticate:") + print(auth_url) + + # Prompt for authorization code + while True: + authorization_code = input("Enter the authorization code: ").strip() + + if not authorization_code: + print("Authorization code cannot be empty. Please try again.") + continue + + self.authorization_code = authorization_code + + try: + self._exchange_code_for_tokens() + logging.info("Successfully authenticated and obtained tokens") + return + except ValueError as e: + print(f"Error exchanging authorization code: {e}") + print("Please try again.") + self.authorization_code = None + + except Exception as e: + logging.error(f"Authentication failed: {e}") + raise ValueError(f"Authentication failed: {e}") + + def fetch_data(self, query: str = None, **kwargs) -> List[DocumentBase]: + """ + Fetch documents from Google Drive based on a query. + + :param query: Search query for files (optional) + :param kwargs: Additional parameters like mime_type, max_results + :return: List of Swarmauri Documents + """ + if not self.access_token: + raise ValueError("Not authenticated. Call authenticate() first.") + + try: + # Prepare request parameters + query_str = query or "" + mime_type = kwargs.get("mime_type", "application/vnd.google-apps.document") + max_results = kwargs.get("max_results", 100) + + # Construct request headers and parameters + headers = { + "Authorization": f"Bearer {self.access_token}", + "Accept": "application/json", + } + + params = { + "q": f"mimeType='{mime_type}' and name contains '{query_str}'", + "pageSize": max_results, + "fields": "files(id,name,mimeType)", + } + + # Make request to Google Drive API + response = self.client.get( + "https://www.googleapis.com/drive/v3/files", + headers=headers, + params=params, + ) + response.raise_for_status() + + files = response.json().get("files", []) + + # Convert Google Drive files to Swarmauri Documents + documents = [] + for file in files: + content = self._get_file_content(file["id"]) + document = DocumentBase( + content=content, + metadata={ + "id": file["id"], + "name": file["name"], + "mime_type": file["mimeType"], + }, + ) + documents.append(document) + + return documents + + except httpx.HTTPError as error: + raise ValueError(f"Error fetching Google Drive files: {error}") + + def _get_file_content(self, file_id: str) -> str: + """ + Retrieve text content from a Google Drive file. + + :param file_id: ID of the Google Drive file + :return: Text content of the file + """ + try: + # Prepare export request + headers = {"Authorization": f"Bearer {self.access_token}"} + + # Export file as plain text + export_url = f"https://www.googleapis.com/drive/v3/files/{file_id}/export" + params = {"mimeType": "text/plain"} + + response = self.client.get(export_url, headers=headers, params=params) + response.raise_for_status() + + return response.text + + except httpx.HTTPError as error: + print(f"An error occurred retrieving file content: {error}") + return "" + + def insert_data(self, data, **kwargs): + """ + Insert a new file into Google Drive. + + :param data: Content of the file to be inserted + :param kwargs: Additional metadata like filename, mime_type + :return: ID of the inserted file + """ + if not self.access_token: + raise ValueError("Not authenticated. Call authenticate() first.") + + try: + headers = { + "Authorization": f"Bearer {self.access_token}", + "Content-Type": "application/json", + } + + # Prepare file metadata + file_metadata = { + "name": kwargs.get("filename", "Untitled Document"), + "mimeType": kwargs.get( + "mime_type", "application/vnd.google-apps.document" + ), + } + + # Prepare file content (base64 encoded) + media_content = base64.b64encode(data.encode("utf-8")).decode("utf-8") + + # Construct payload + payload = { + "metadata": file_metadata, + "media": {"mimeType": "text/plain", "body": media_content}, + } + + # Make request to create file + response = self.client.post( + "https://www.googleapis.com/upload/drive/v3/files", + headers=headers, + json=payload, + ) + response.raise_for_status() + + return response.json().get("id") + + except httpx.HTTPError as error: + raise ValueError(f"Error inserting file: {error}") + + def update_data(self, identifier, data, **kwargs): + """ + Update an existing Google Drive file. + + :param identifier: File ID to update + :param data: New content for the file + :param kwargs: Additional update parameters + """ + if not self.access_token: + raise ValueError("Not authenticated. Call authenticate() first.") + + try: + headers = { + "Authorization": f"Bearer {self.access_token}", + "Content-Type": "application/json", + } + + # Prepare file content (base64 encoded) + media_content = base64.b64encode(data.encode("utf-8")).decode("utf-8") + + # Construct payload + payload = {"media": {"mimeType": "text/plain", "body": media_content}} + + # Make request to update file + response = self.client.patch( + f"https://www.googleapis.com/upload/drive/v3/files/{identifier}", + headers=headers, + json=payload, + ) + response.raise_for_status() + + except httpx.HTTPError as error: + raise ValueError(f"Error updating file: {error}") + + def delete_data(self, identifier, **kwargs): + """ + Delete a file from Google Drive. + + :param identifier: File ID to delete + """ + if not self.access_token: + raise ValueError("Not authenticated. Call authenticate() first.") + + try: + headers = {"Authorization": f"Bearer {self.access_token}"} + + response = self.client.delete( + f"https://www.googleapis.com/drive/v3/files/{identifier}", + headers=headers, + ) + response.raise_for_status() + + except httpx.HTTPError as error: + raise ValueError(f"Error deleting file: {error}") + + def test_connection(self, **kwargs): + """ + Test the connection to Google Drive by listing files. + + :return: Boolean indicating connection success + """ + try: + if not self.access_token: + self.authenticate(**kwargs) + + # Prepare request headers + headers = { + "Authorization": f"Bearer {self.access_token}", + "Accept": "application/json", + } + + # List first 5 files to test connection + params = {"pageSize": 5, "fields": "files(id,name)"} + + response = self.client.get( + "https://www.googleapis.com/drive/v3/files", + headers=headers, + params=params, + ) + response.raise_for_status() + + files = response.json().get("files", []) + return len(files) > 0 + + except Exception as e: + print(f"Connection test failed: {e}") + return False diff --git a/pkgs/swarmauri/swarmauri/dataconnectors/concrete/__init__.py b/pkgs/swarmauri/swarmauri/dataconnectors/concrete/__init__.py new file mode 100644 index 000000000..612d7f2e4 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/dataconnectors/concrete/__init__.py @@ -0,0 +1,16 @@ +from swarmauri.utils._lazy_import import _lazy_import + +# List of data connector files names (file names without the ".py" extension) and corresponding class names +data_connector_files = [ + ( + "swarmauri.dataconnectors.concrete.GoogleDriveDataConnector", + "GoogleDriveDataConnector", + ), +] + +# Lazy loading of data connector classes, storing them in variables +for module_name, class_name in data_connector_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded data connector classes to __all__ +__all__ = [class_name for _, class_name in data_connector_files] diff --git a/pkgs/swarmauri/tests/static/credentials.json b/pkgs/swarmauri/tests/static/credentials.json new file mode 100644 index 000000000..a36c5e36b --- /dev/null +++ b/pkgs/swarmauri/tests/static/credentials.json @@ -0,0 +1,5 @@ +{ + "client_id": "Put your client_id here", + "client_secret": "Put your client_secret here", + "redirect_uri": "put your redirect_uri here or use :ietf:wg:oauth:2.0:oob" +} \ No newline at end of file diff --git a/pkgs/swarmauri/tests/unit/dataconnectors/GoogleDriveDataConnector_unit_test.py b/pkgs/swarmauri/tests/unit/dataconnectors/GoogleDriveDataConnector_unit_test.py new file mode 100644 index 000000000..9b1a092da --- /dev/null +++ b/pkgs/swarmauri/tests/unit/dataconnectors/GoogleDriveDataConnector_unit_test.py @@ -0,0 +1,86 @@ +import pytest +from swarmauri.dataconnectors.concrete.GoogleDriveDataConnector import ( + GoogleDriveDataConnector, +) + + +@pytest.fixture(scope="module") +def authenticated_connector(): + """Authenticate the GoogleDriveDataConnector once for the test suite.""" + # Path to the valid credentials JSON file + credentials_path = "pkgs/swarmauri/tests/static/credentials.json" + connector = GoogleDriveDataConnector(credentials_path=credentials_path) + + # Perform authentication once + try: + connector.authenticate() # Requires manual input for the authorization code + except Exception as e: + pytest.fail(f"Authentication failed: {e}") + + return connector + + +@pytest.fixture(scope="module") +def shared_file_id(): + """Return a shared file ID for testing.""" + return {} + + +@pytest.mark.skip(reason="Skipping test_generate_authorization_url") +def test_generate_authorization_url(): + """Test generate_authorization_url without authentication.""" + # Path to the valid credentials JSON file + credentials_path = "pkgs/swarmauri/tests/static/credentials.json" + connector = GoogleDriveDataConnector(credentials_path=credentials_path) + url = connector.generate_authorization_url() + assert isinstance(url, str) + assert "client_id" in url + assert "redirect_uri" in url + assert "https://accounts.google.com/o/oauth2/v2/auth" in url + + +@pytest.mark.skip(reason="Skipping test_fetch_data") +def test_fetch_data(authenticated_connector): + """Test fetching data from Google Drive.""" + documents = authenticated_connector.fetch_data(query="test") + assert isinstance(documents, list) + if documents: + assert all(hasattr(doc, "content") for doc in documents) + assert all(hasattr(doc, "metadata") for doc in documents) + + +@pytest.mark.skip(reason="Skipping test_insert_data") +def test_insert_data(authenticated_connector, shared_file_id): + """Test inserting data into Google Drive.""" + test_data = "Sample content for Google Drive file" + file_id = authenticated_connector.insert_data(test_data, filename="test_file.txt") + assert isinstance(file_id, str) + shared_file_id["file_id"] = file_id + + +@pytest.mark.skip(reason="Skipping test_update_data") +def test_update_data(authenticated_connector, shared_file_id): + """Test updating data in Google Drive.""" + file_id = shared_file_id["file_id"] + updated_content = "Updated content for Google Drive file" + try: + authenticated_connector.update_data(file_id, updated_content) + except Exception as e: + pytest.fail(f"Failed to update file: {e}") + + +@pytest.mark.skip(reason="Skipping test_delete_data") +def test_delete_data(authenticated_connector, shared_file_id): + """Test deleting data from Google Drive.""" + file_id = shared_file_id["file_id"] # Replace with an actual file ID + try: + authenticated_connector.delete_data(file_id) + except Exception as e: + pytest.fail(f"Failed to delete file: {e}") + + +@pytest.mark.skip(reason="Skipping test_connection") +def test_connection(authenticated_connector): + """Test the connection to Google Drive.""" + connection_success = authenticated_connector.test_connection() + assert connection_success is True From 21ded19cf9928a918e8feb4fb721a796ae875f6c Mon Sep 17 00:00:00 2001 From: michaeldecent2 <111002205+MichaelDecent@users.noreply.github.com> Date: Mon, 9 Dec 2024 11:59:58 +0100 Subject: [PATCH 047/168] swarm - Add MethodSignatureExtractor and update Parameter type to support lists --- .../swarmauri/tools/concrete/Parameter.py | 9 +- .../method_signature_extractor_decorator.py | 113 ++++++++++++ ...thod_signature_extractor_decorator_test.py | 172 ++++++++++++++++++ 3 files changed, 291 insertions(+), 3 deletions(-) create mode 100644 pkgs/swarmauri/swarmauri/utils/method_signature_extractor_decorator.py create mode 100644 pkgs/swarmauri/tests/unit/utils/method_signature_extractor_decorator_test.py diff --git a/pkgs/swarmauri/swarmauri/tools/concrete/Parameter.py b/pkgs/swarmauri/swarmauri/tools/concrete/Parameter.py index 2f5a357b6..59afe9a83 100644 --- a/pkgs/swarmauri/swarmauri/tools/concrete/Parameter.py +++ b/pkgs/swarmauri/swarmauri/tools/concrete/Parameter.py @@ -1,7 +1,10 @@ -from typing import Literal, Union -from pydantic import Field +from typing import List, Literal, Union from swarmauri.tools.base.ParameterBase import ParameterBase class Parameter(ParameterBase): - type: Union[Literal["string", "number", "boolean", "array", "object"], str] + type: Union[ + Literal["string", "number", "boolean", "array", "object"], + str, + List[str], + ] diff --git a/pkgs/swarmauri/swarmauri/utils/method_signature_extractor_decorator.py b/pkgs/swarmauri/swarmauri/utils/method_signature_extractor_decorator.py new file mode 100644 index 000000000..20375df83 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/utils/method_signature_extractor_decorator.py @@ -0,0 +1,113 @@ +from typing import ( + List, + Any, + Union, + Optional, + Callable, + get_type_hints, + get_args, + get_origin, +) +import inspect +from functools import wraps +from pydantic import BaseModel +from swarmauri.tools.concrete.Parameter import Parameter + + +class MethodSignatureExtractor(BaseModel): + parameters: List[Parameter] = [] + method: Callable + _type_mapping: dict = { + int: "integer", + float: "number", + str: "string", + bool: "boolean", + list: "array", + dict: "object", + Any: "any", + } + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.parameters = self.extract_signature_details() + + def _python_type_to_json_schema_type(self, py_type): + if get_origin(py_type) is not None: + origin = get_origin(py_type) + args = get_args(py_type) + + if origin is list: + items_type = self._python_type_to_json_schema_type(args[0]) + return {"type": "array", "items": items_type} + elif origin is dict: + return {"type": "object"} + elif origin in (Union, Optional): + if len(args) == 2 and type(None) in args: + non_none_type = args[0] if args[1] is type(None) else args[1] + return self._python_type_to_json_schema_type(non_none_type) + return { + "oneOf": [ + self._python_type_to_json_schema_type(arg) for arg in args + ] + } + return {"type": self._type_mapping.get(origin, "string")} + else: + return {"type": self._type_mapping.get(py_type, "string")} + + def extract_signature_details(self): + sig = inspect.signature(self.method) + type_hints = get_type_hints(self.method) + parameters = sig.parameters + details_list = [] + for param_name, param in parameters.items(): + if param_name == "self": + continue + + param_type = type_hints.get(param_name, Any) + param_default = ( + param.default if param.default is not inspect.Parameter.empty else None + ) + required = param.default is inspect.Parameter.empty + enum = None + param_type_json_schema = self._python_type_to_json_schema_type(param_type) + print(param_type_json_schema) + + if "oneOf" in param_type_json_schema: + param_type_json_schema["type"] = [ + type_["type"] for type_ in param_type_json_schema["oneOf"] + ] + + description = f"Parameter {param_name} of type {param_type_json_schema}" + + detail = Parameter( + name=param_name, + type=param_type_json_schema["type"], + description=description, + required=required, + enum=enum, + ) + details_list.append(detail) + + return details_list + + +def extract_method_signature(func: Callable): + """ + A decorator that extracts method signature details and attaches them to the function. + + Args: + func (Callable): The function to extract signature details for. + + Returns: + Callable: The original function with added signature_details attribute. + """ + + @wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + extractor = MethodSignatureExtractor(method=func) + + wrapper.signature_details = extractor.parameters + + return wrapper diff --git a/pkgs/swarmauri/tests/unit/utils/method_signature_extractor_decorator_test.py b/pkgs/swarmauri/tests/unit/utils/method_signature_extractor_decorator_test.py new file mode 100644 index 000000000..ea14ca385 --- /dev/null +++ b/pkgs/swarmauri/tests/unit/utils/method_signature_extractor_decorator_test.py @@ -0,0 +1,172 @@ +from typing import List, Optional, Union, Dict, Any +from swarmauri.utils.method_signature_extractor_decorator import ( + MethodSignatureExtractor, + extract_method_signature, +) + + +# Test functions with various signature types +def simple_function(x: int, y: str): + return x, y + + +def optional_function(a: Optional[int] = None, b: str = "default"): + return a, b + + +def list_function(items: List[str], count: int = 1): + return items, count + + +def union_function(value: Union[int, str]): + return value + + +def complex_function( + x: int, y: Optional[List[float]] = None, z: Union[str, Dict[str, Any]] = "default" +): + return x, y, z + + +class TestMethodSignatureExtractor: + def test_simple_function_extraction(self): + """Test extraction of a simple function with basic types""" + extractor = MethodSignatureExtractor(method=simple_function) + + assert len(extractor.parameters) == 2 + + # Check first parameter + assert extractor.parameters[0].name == "x" + assert extractor.parameters[0].type == "integer" + assert extractor.parameters[0].required is True + + # Check second parameter + assert extractor.parameters[1].name == "y" + assert extractor.parameters[1].type == "string" + assert extractor.parameters[1].required is True + + def test_optional_function_extraction(self): + """Test extraction of a function with optional parameters""" + extractor = MethodSignatureExtractor(method=optional_function) + + assert len(extractor.parameters) == 2 + + # Check first parameter (optional) + assert extractor.parameters[0].name == "a" + assert extractor.parameters[0].type == "integer" + assert extractor.parameters[0].required is False + + # Check second parameter (with default) + assert extractor.parameters[1].name == "b" + assert extractor.parameters[1].type == "string" + assert extractor.parameters[1].required is False + + def test_list_function_extraction(self): + """Test extraction of a function with list parameter""" + extractor = MethodSignatureExtractor(method=list_function) + + assert len(extractor.parameters) == 2 + + # Check first parameter (list) + assert extractor.parameters[0].name == "items" + assert extractor.parameters[0].type == "array" + assert extractor.parameters[0].required is True + + # Check second parameter (with default) + assert extractor.parameters[1].name == "count" + assert extractor.parameters[1].type == "integer" + assert extractor.parameters[1].required is False + + def test_union_function_extraction(self): + """Test extraction of a function with union type""" + extractor = MethodSignatureExtractor(method=union_function) + + assert len(extractor.parameters) == 1 + + # Check union parameter + assert extractor.parameters[0].name == "value" + assert extractor.parameters[0].type is not None + assert len(extractor.parameters[0].type) == 2 + + def test_complex_function_extraction(self): + """Test extraction of a function with multiple complex types""" + extractor = MethodSignatureExtractor(method=complex_function) + + assert len(extractor.parameters) == 3 + + # Check first parameter + assert extractor.parameters[0].name == "x" + assert extractor.parameters[0].type == "integer" + assert extractor.parameters[0].required is True + + # Check second parameter (optional list) + assert extractor.parameters[1].name == "y" + assert extractor.parameters[1].type == "array" + assert extractor.parameters[1].required is False + + # Check third parameter (union type with default) + assert extractor.parameters[2].name == "z" + assert extractor.parameters[2].type is not None + assert extractor.parameters[2].required is False + + def test_decorator_signature_extraction(self): + """Test the extract_method_signature decorator""" + + @extract_method_signature + def test_decorator_func(a: int, b: Optional[str] = None): + pass + + # Check if signature_details is added to the function + assert hasattr(test_decorator_func, "signature_details") + + # Verify the details + details = test_decorator_func.signature_details + assert len(details) == 2 + + # First parameter + assert details[0].name == "a" + assert details[0].type == "integer" + assert details[0].required is True + + # Second parameter + assert details[1].name == "b" + assert details[1].type == "string" + assert details[1].required is False + + def test_type_mapping(self): + """Test the type mapping functionality""" + extractor = MethodSignatureExtractor(method=simple_function) + + # Check predefined type mappings + type_mapping = extractor._type_mapping + assert type_mapping[int] == "integer" + assert type_mapping[float] == "number" + assert type_mapping[str] == "string" + assert type_mapping[bool] == "boolean" + assert type_mapping[list] == "array" + assert type_mapping[dict] == "object" + assert type_mapping[Any] == "any" + + +# Additional edge case tests +def test_empty_function(): + """Test function with no parameters""" + + def empty_func(): + pass + + extractor = MethodSignatureExtractor(method=empty_func) + assert len(extractor.parameters) == 0 + + +def test_method_with_self(): + """Test method of a class with self parameter""" + + class TestClass: + def method(self, x: int): + return x + + extractor = MethodSignatureExtractor(method=TestClass.method) + assert len(extractor.parameters) == 1 + assert extractor.parameters[0].name == "x" + assert extractor.parameters[0].type == "integer" From a3b504ab6db26dde2f811a427dc8e157690f1446 Mon Sep 17 00:00:00 2001 From: 3rdSon Date: Tue, 10 Dec 2024 09:06:52 +0100 Subject: [PATCH 048/168] implemented states --- .../swarmauri_community/state/__init__.py | 1 + .../state/concrete/ClipboardState.py | 69 ++++++++++++++ .../state/concrete/__init__.py | 10 ++ .../tests/unit/state/ClipboardState_test.py | 91 +++++++++++++++++++ pkgs/core/swarmauri_core/state/IState.py | 28 ++++++ pkgs/swarmauri/swarmauri/state/__init__.py | 1 + .../swarmauri/state/base/StateBase.py | 47 ++++++++++ .../swarmauri/state/base/__init__.py | 0 .../swarmauri/state/concrete/DictState.py | 50 ++++++++++ .../swarmauri/state/concrete/__init__.py | 13 +++ .../tests/unit/state/DictState_unit_test.py | 85 +++++++++++++++++ 11 files changed, 395 insertions(+) create mode 100644 pkgs/community/swarmauri_community/state/__init__.py create mode 100644 pkgs/community/swarmauri_community/state/concrete/ClipboardState.py create mode 100644 pkgs/community/swarmauri_community/state/concrete/__init__.py create mode 100644 pkgs/community/tests/unit/state/ClipboardState_test.py create mode 100644 pkgs/core/swarmauri_core/state/IState.py create mode 100644 pkgs/swarmauri/swarmauri/state/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/state/base/StateBase.py create mode 100644 pkgs/swarmauri/swarmauri/state/base/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/state/concrete/DictState.py create mode 100644 pkgs/swarmauri/swarmauri/state/concrete/__init__.py create mode 100644 pkgs/swarmauri/tests/unit/state/DictState_unit_test.py diff --git a/pkgs/community/swarmauri_community/state/__init__.py b/pkgs/community/swarmauri_community/state/__init__.py new file mode 100644 index 000000000..3828350c1 --- /dev/null +++ b/pkgs/community/swarmauri_community/state/__init__.py @@ -0,0 +1 @@ +from swarmauri_community.state.concrete import * diff --git a/pkgs/community/swarmauri_community/state/concrete/ClipboardState.py b/pkgs/community/swarmauri_community/state/concrete/ClipboardState.py new file mode 100644 index 000000000..53f73bcde --- /dev/null +++ b/pkgs/community/swarmauri_community/state/concrete/ClipboardState.py @@ -0,0 +1,69 @@ +import pyperclip +from typing import Dict, Any +from swarmauri.state.base.StateBase import StateBase + + +class ClipboardState(StateBase): + """ + A concrete implementation of StateBase that uses the system clipboard to store and retrieve state data. + """ + + def read(self) -> Dict[str, Any]: + """ + Reads the current state from the clipboard as a dictionary. + """ + try: + clipboard_content = pyperclip.paste() + # Ensure the clipboard contains valid data (e.g., a JSON string that can be parsed) + if clipboard_content: + return eval( + clipboard_content + ) # Replace eval with JSON for safer parsing + return {} + except Exception as e: + raise ValueError(f"Failed to read state from clipboard: {e}") + + def write(self, data: Dict[str, Any]) -> None: + """ + Replaces the current state with the given data by copying it to the clipboard. + """ + try: + pyperclip.copy( + str(data) + ) # Convert dictionary to string for clipboard storage + except Exception as e: + raise ValueError(f"Failed to write state to clipboard: {e}") + + def update(self, data: Dict[str, Any]) -> None: + """ + Updates the current state with the given data by merging with clipboard content. + """ + try: + current_state = self.read() + current_state.update(data) + self.write(current_state) + except Exception as e: + raise ValueError(f"Failed to update state on clipboard: {e}") + + def reset(self) -> None: + """ + Resets the clipboard state to an empty dictionary. + """ + try: + self.write({}) + except Exception as e: + raise ValueError(f"Failed to reset clipboard state: {e}") + + def deep_copy(self) -> "ClipboardState": + """ + Creates a deep copy of the current state. In this context, simply returns a new ClipboardState with the same clipboard data. + """ + try: + current_state = self.read() + new_instance = ClipboardState() + new_instance.write(current_state) + return new_instance + except Exception as e: + raise ValueError( + f"Failed to create a deep copy of the clipboard state: {e}" + ) diff --git a/pkgs/community/swarmauri_community/state/concrete/__init__.py b/pkgs/community/swarmauri_community/state/concrete/__init__.py new file mode 100644 index 000000000..92405b4a9 --- /dev/null +++ b/pkgs/community/swarmauri_community/state/concrete/__init__.py @@ -0,0 +1,10 @@ +from swarmauri.utils._lazy_import import _lazy_import + +state_files = [ + ("swarmauri_community.state.concrete.ClipboardState", "ClipboardState"), +] + +for module_name, class_name in state_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +__all__ = [class_name for _, class_name in state_files] diff --git a/pkgs/community/tests/unit/state/ClipboardState_test.py b/pkgs/community/tests/unit/state/ClipboardState_test.py new file mode 100644 index 000000000..2b9d200d6 --- /dev/null +++ b/pkgs/community/tests/unit/state/ClipboardState_test.py @@ -0,0 +1,91 @@ +import pytest +import pyperclip +from swarmauri_community.state.concrete.ClipboardState import ClipboardState + + +@pytest.fixture +def clipboard_state(): + """ + Fixture to create a ClipboardState instance and clean up clipboard after tests. + """ + # Store original clipboard content + original_clipboard = pyperclip.paste() + + # Create ClipboardState + state = ClipboardState() + + # Yield the state for tests to use + yield state + + # Restore original clipboard content after tests + pyperclip.copy(original_clipboard) + + +@pytest.mark.unit +def test_ubc_resource(clipboard_state): + """ + Test the resource type of the ClipboardState. + """ + assert clipboard_state.resource == "State" + + +@pytest.mark.unit +def test_write_and_read(clipboard_state): + """ + Test writing data to clipboard and reading it back. + """ + test_data = {"key1": "value1", "key2": 42} + clipboard_state.write(test_data) + read_data = clipboard_state.read() + assert read_data == test_data + + +@pytest.mark.unit +def test_update(clipboard_state): + """ + Test updating existing clipboard data. + """ + # Initial write + initial_data = {"existing_key": "existing_value"} + clipboard_state.write(initial_data) + + # Update with new data + update_data = {"new_key": "new_value"} + clipboard_state.update(update_data) + + # Read and verify merged data + read_data = clipboard_state.read() + expected_data = {"existing_key": "existing_value", "new_key": "new_value"} + assert read_data == expected_data + + +@pytest.mark.unit +def test_reset(clipboard_state): + """ + Test resetting the clipboard state to an empty dictionary. + """ + # Write some data + clipboard_state.write({"some_key": "some_value"}) + + # Reset + clipboard_state.reset() + + # Verify empty state + assert clipboard_state.read() == {} + + +@pytest.mark.unit +def test_deep_copy(clipboard_state): + """ + Test creating a deep copy of the clipboard state. + """ + # Write initial data + initial_data = {"key1": "value1", "key2": "value2"} + clipboard_state.write(initial_data) + + # Create deep copy + copied_state = clipboard_state.deep_copy() + + # Verify copied state + assert isinstance(copied_state, ClipboardState) + assert copied_state.read() == initial_data diff --git a/pkgs/core/swarmauri_core/state/IState.py b/pkgs/core/swarmauri_core/state/IState.py new file mode 100644 index 000000000..02c571bea --- /dev/null +++ b/pkgs/core/swarmauri_core/state/IState.py @@ -0,0 +1,28 @@ +from abc import ABC, abstractmethod +from typing import Dict, Any + + +class IState(ABC): + @abstractmethod + def read(self) -> Dict[str, Any]: + """ + Reads and returns the current state as a dictionary. + """ + + @abstractmethod + def write(self, data: Dict[str, Any]) -> None: + """ + Replaces the current state with the given data. + """ + + @abstractmethod + def update(self, data: Dict[str, Any]) -> None: + """ + Updates the state with the given data. + """ + + @abstractmethod + def reset(self) -> None: + """ + Resets the state to its initial state. + """ diff --git a/pkgs/swarmauri/swarmauri/state/__init__.py b/pkgs/swarmauri/swarmauri/state/__init__.py new file mode 100644 index 000000000..67f09d1f4 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/state/__init__.py @@ -0,0 +1 @@ +from swarmauri.state.concrete import * diff --git a/pkgs/swarmauri/swarmauri/state/base/StateBase.py b/pkgs/swarmauri/swarmauri/state/base/StateBase.py new file mode 100644 index 000000000..03aa0f544 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/state/base/StateBase.py @@ -0,0 +1,47 @@ +from typing import Dict, Any, Optional, Literal +from pydantic import Field, ConfigDict +from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes +from swarmauri_core.state.IState import IState + + +class StateBase(IState, ComponentBase): + """ + Abstract base class for state management, extending IState and ComponentBase. + """ + + state_data: Dict[str, Any] = Field( + default_factory=dict, description="The current state data." + ) + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + resource: Optional[str] = Field(default=ResourceTypes.STATE.value, frozen=True) + type: Literal["StateBase"] = "StateBase" + + def read(self) -> Dict[str, Any]: + """ + Reads and returns the current state as a dictionary. + """ + raise NotImplementedError("Subclasses must implement 'read'.") + + def write(self, data: Dict[str, Any]) -> None: + """ + Replaces the current state with the given data. + """ + raise NotImplementedError("Subclasses must implement 'write'.") + + def update(self, data: Dict[str, Any]) -> None: + """ + Updates the state with the given data. + """ + raise NotImplementedError("Subclasses must implement 'update'.") + + def reset(self) -> None: + """ + Resets the state to its initial state. + """ + raise NotImplementedError("Subclasses must implement 'reset'.") + + def deep_copy(self) -> "IState": + """ + Creates a deep copy of the current state. + """ + raise NotImplementedError("Subclasses must implement 'deep_copy'.") diff --git a/pkgs/swarmauri/swarmauri/state/base/__init__.py b/pkgs/swarmauri/swarmauri/state/base/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/state/concrete/DictState.py b/pkgs/swarmauri/swarmauri/state/concrete/DictState.py new file mode 100644 index 000000000..1f5c6adee --- /dev/null +++ b/pkgs/swarmauri/swarmauri/state/concrete/DictState.py @@ -0,0 +1,50 @@ +from typing import Dict, Any +from copy import deepcopy +from pydantic import Field, model_validator +from swarmauri.state.base.StateBase import StateBase + + +class DictState(StateBase): + """ + A concrete implementation of StateBase that manages state as a dictionary. + """ + + state_data: Dict[str, Any] = Field( + default_factory=dict, description="The current state data." + ) + + def read(self) -> Dict[str, Any]: + """ + Reads and returns the current state as a dictionary. + """ + return deepcopy(self.state_data) + + def write(self, data: Dict[str, Any]) -> None: + """ + Replaces the current state with the given data. + """ + self.state_data = deepcopy(data) + + def update(self, data: Dict[str, Any]) -> None: + """ + Updates the state with the given data. + """ + self.state_data.update(data) + + def reset(self) -> None: + """ + Resets the state to an empty dictionary. + """ + self.state_data = {} + + def deep_copy(self) -> "DictState": + """ + Creates a deep copy of the current state. + """ + return DictState(state_data=deepcopy(self.state_data)) + + @model_validator(mode="after") + def _ensure_deep_copy(self): + # Ensures that state_data is always a deep copy + self.state_data = deepcopy(self.state_data) + return self diff --git a/pkgs/swarmauri/swarmauri/state/concrete/__init__.py b/pkgs/swarmauri/swarmauri/state/concrete/__init__.py new file mode 100644 index 000000000..3752dc4af --- /dev/null +++ b/pkgs/swarmauri/swarmauri/state/concrete/__init__.py @@ -0,0 +1,13 @@ +from swarmauri.utils._lazy_import import _lazy_import + +# List of state names (file names without the ".py" extension) and corresponding class names +state_files = [ + ("swarmauri.state.concrete.DictState", "DictState"), +] + +# Lazy loading of state classes, storing them in variables +for module_name, class_name in state_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded state classes to __all__ +__all__ = [class_name for _, class_name in state_files] diff --git a/pkgs/swarmauri/tests/unit/state/DictState_unit_test.py b/pkgs/swarmauri/tests/unit/state/DictState_unit_test.py new file mode 100644 index 000000000..8450efb47 --- /dev/null +++ b/pkgs/swarmauri/tests/unit/state/DictState_unit_test.py @@ -0,0 +1,85 @@ +import pytest +from swarmauri.state.concrete.DictState import DictState + + +@pytest.fixture +def dict_state(): + """ + Fixture to create a DictState instance for testing. + """ + # Create DictState + state = DictState() + + # Yield the state for tests to use + yield state + + +@pytest.mark.unit +def test_resource_type(dict_state): + """ + Test the resource type of the DictState. + """ + assert dict_state.resource == "State" + + +@pytest.mark.unit +def test_write_and_read(dict_state): + """ + Test writing data to DictState and reading it back. + """ + test_data = {"key1": "value1", "key2": 42} + dict_state.write(test_data) + read_data = dict_state.read() + assert read_data == test_data + + +@pytest.mark.unit +def test_update(dict_state): + """ + Test updating existing DictState data. + """ + # Initial write + initial_data = {"existing_key": "existing_value"} + dict_state.write(initial_data) + + # Update with new data + update_data = {"new_key": "new_value"} + dict_state.update(update_data) + + # Read and verify merged data + read_data = dict_state.read() + expected_data = {"existing_key": "existing_value", "new_key": "new_value"} + assert read_data == expected_data + + +@pytest.mark.unit +def test_reset(dict_state): + """ + Test resetting the DictState to an empty dictionary. + """ + # Write some data + dict_state.write({"some_key": "some_value"}) + + # Reset + dict_state.reset() + + # Verify empty state + assert dict_state.read() == {} + + +@pytest.mark.unit +def test_deep_copy(dict_state): + # Write initial data + initial_data = {"key1": "value1", "key2": "value2"} + dict_state.write(initial_data) + + # Create deep copy + copied_state = dict_state.deep_copy() + + # Verify copied state + assert isinstance(copied_state, DictState) + assert copied_state.read() == initial_data + + # Verify deep copy by modifying original and copy independently + dict_state.update({"new_key": "new_value"}) + assert copied_state.read() == initial_data # Copy should remain unchanged From f0ff92a82c89375b05b20af6fbbb9ac12eeb2a79 Mon Sep 17 00:00:00 2001 From: michaeldecent2 <111002205+MichaelDecent@users.noreply.github.com> Date: Tue, 10 Dec 2024 16:57:22 +0100 Subject: [PATCH 049/168] feat: implement factory pattern with base and agent factories; add JSON validation methods --- pkgs/core/swarmauri_core/ComponentBase.py | 69 ++++++++++++++++++- .../agent_factories/IAgentFactory.py | 2 - .../core/swarmauri_core/factories/IFactory.py | 21 ++++++ .../core/swarmauri_core/factories/__init__.py | 0 .../swarmauri/swarmauri/factories/__init__.py | 0 .../swarmauri/factories/base/FactoryBase.py | 20 ++++++ .../swarmauri/factories/base/_init__.py | 0 .../factories/concrete/AgentFactory.py | 30 ++++++++ .../swarmauri/factories/concrete/Factory.py | 40 +++++++++++ .../swarmauri/factories/concrete/__init__.py | 0 .../unit/factories/AgentFactory_unit_test.py | 35 ++++++++++ .../tests/unit/factories/Factory_unit_test.py | 50 ++++++++++++++ 12 files changed, 262 insertions(+), 5 deletions(-) create mode 100644 pkgs/core/swarmauri_core/factories/IFactory.py create mode 100644 pkgs/core/swarmauri_core/factories/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/factories/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py create mode 100644 pkgs/swarmauri/swarmauri/factories/base/_init__.py create mode 100644 pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py create mode 100644 pkgs/swarmauri/swarmauri/factories/concrete/Factory.py create mode 100644 pkgs/swarmauri/swarmauri/factories/concrete/__init__.py create mode 100644 pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py create mode 100644 pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py diff --git a/pkgs/core/swarmauri_core/ComponentBase.py b/pkgs/core/swarmauri_core/ComponentBase.py index 13a096001..a86c8b4fb 100644 --- a/pkgs/core/swarmauri_core/ComponentBase.py +++ b/pkgs/core/swarmauri_core/ComponentBase.py @@ -1,12 +1,13 @@ +import json from typing import ( + Any, + Dict, Optional, List, Literal, TypeVar, Type, Union, - Annotated, - Generic, ClassVar, Set, get_args, @@ -16,11 +17,14 @@ from enum import Enum import inspect import hashlib -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field, ValidationError, field_validator import logging from swarmauri_core.typing import SubclassUnion +T = TypeVar("T", bound="ComponentBase") + + class ResourceTypes(Enum): UNIVERSAL_BASE = "ComponentBase" AGENT = "Agent" @@ -180,3 +184,62 @@ def swm_path(self): @property def swm_isremote(self): return bool(self.host) + + @classmethod + def model_validate_json( + cls: Type[T], json_payload: Union[str, Dict[str, Any]], strict: bool = False + ) -> T: + # Ensure we're working with a dictionary + if isinstance(json_payload, str): + try: + payload_dict = json.loads(json_payload) + except json.JSONDecodeError: + raise ValueError("Invalid JSON payload") + else: + payload_dict = json_payload + + # Try to determine the specific component type + component_type = payload_dict.get("type", "ComponentBase") + + # Attempt to find the correct subclass + target_cls = cls.get_subclass_by_type(component_type) + + # Fallback logic + if target_cls is None: + if strict: + raise ValueError(f"Cannot resolve component type: {component_type}") + target_cls = cls + logging.warning( + f"Falling back to base ComponentBase for type: {component_type}" + ) + + # Validate using the determined class + try: + return target_cls.model_validate(payload_dict) + except ValidationError as e: + logging.error(f"Validation failed for {component_type}: {e}") + raise + + @classmethod + def get_subclass_by_type(cls, type_name: str) -> Optional[Type["ComponentBase"]]: + # First, check for exact match in registered subclasses + for subclass in cls.__swm_subclasses__: + if ( + subclass.__name__ == type_name + or getattr(subclass, "type", None) == type_name + ): + return subclass + + # If no exact match, try case-insensitive search + for subclass in cls.__swm_subclasses__: + if ( + subclass.__name__.lower() == type_name.lower() + or str(getattr(subclass, "type", "")).lower() == type_name.lower() + ): + return subclass + + return None + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "ComponentBase": + return cls.model_validate(data) diff --git a/pkgs/core/swarmauri_core/agent_factories/IAgentFactory.py b/pkgs/core/swarmauri_core/agent_factories/IAgentFactory.py index 8dc24b3d8..bbd75b2ce 100644 --- a/pkgs/core/swarmauri_core/agent_factories/IAgentFactory.py +++ b/pkgs/core/swarmauri_core/agent_factories/IAgentFactory.py @@ -75,5 +75,3 @@ def __hash__(self): but it's declared here to indicate that implementing classes must provide it. """ pass - - \ No newline at end of file diff --git a/pkgs/core/swarmauri_core/factories/IFactory.py b/pkgs/core/swarmauri_core/factories/IFactory.py new file mode 100644 index 000000000..eed40b31b --- /dev/null +++ b/pkgs/core/swarmauri_core/factories/IFactory.py @@ -0,0 +1,21 @@ +from abc import ABC, abstractmethod +from typing import Any, Generic, Type, TypeVar + +# Generic type variable for flexible factory implementations +T = TypeVar("T") + + +class IFactory(ABC, Generic[T]): + """ + Interface defining core methods for factories. + """ + + @abstractmethod + def create(self, *args: Any, **kwargs: Any) -> T: + """Create and return an instance of type T.""" + pass + + @abstractmethod + def register(self, resource: str, name: str, resource_class: Type[T]) -> None: + """Register a class with the factory.""" + pass diff --git a/pkgs/core/swarmauri_core/factories/__init__.py b/pkgs/core/swarmauri_core/factories/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/factories/__init__.py b/pkgs/swarmauri/swarmauri/factories/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py b/pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py new file mode 100644 index 000000000..c4748c543 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py @@ -0,0 +1,20 @@ +from typing import Any, Type +from swarmauri_core.factories.IFactory import IFactory, T + + +class FactoryBase(IFactory[T]): + """ + Base factory class for registering and creating instances. + """ + + def register(self, resource: str, type: str, resource_class: Type[T]) -> None: + """ + Register a resource class under a specific resource and type. + """ + raise NotImplementedError("register method must be implemented in derived classes.") + + def create(self, resource: str, type: str, *args: Any, **kwargs: Any) -> T: + """ + Create an instance of the class associated with the given resource and type. + """ + raise NotImplementedError("create method must be implemented in derived classes.") \ No newline at end of file diff --git a/pkgs/swarmauri/swarmauri/factories/base/_init__.py b/pkgs/swarmauri/swarmauri/factories/base/_init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py b/pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py new file mode 100644 index 000000000..16f513870 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py @@ -0,0 +1,30 @@ +from typing import Any, Dict, Type +from swarmauri.factories.base.FactoryBase import FactoryBase, T + + +class AgentFactory(FactoryBase[T]): + """ + Class-specific factory for managing resources and types. + """ + + def __init__(self) -> None: + # Make _registry an instance-level variable. + self._registry: Dict[str, Type[T]] = {} + + def register(self, type: str, resource_class: Type[T]) -> None: + """ + Register a resource class with a specific type. + """ + if type in self._registry: + raise ValueError(f"Type '{type}' is already registered.") + self._registry[type] = resource_class + + def create(self, type: str, *args: Any, **kwargs: Any) -> T: + """ + Create an instance of the class associated with the given type name. + """ + if type not in self._registry: + raise ValueError(f"Type '{type}' is not registered.") + + cls = self._registry[type] + return cls(*args, **kwargs) diff --git a/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py b/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py new file mode 100644 index 000000000..1be46451b --- /dev/null +++ b/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py @@ -0,0 +1,40 @@ +from typing import Any, Dict, Type +from swarmauri.factories.base.FactoryBase import FactoryBase, T + + +class Factory(FactoryBase[T]): + """ + Non-recursive factory extending FactoryBase. + """ + def __init__(self) -> None: + # Make _resource_registry an instance-level variable. + self._resource_registry: Dict[str, Dict[str, Type[T]]] = {} + + def register(self, resource: str, type: str, resource_class: Type[T]) -> None: + """ + Register a resource class under a specific resource and type. + """ + if resource not in self._resource_registry: + self._resource_registry[resource] = {} + + if type in self._resource_registry[resource]: + raise ValueError( + f"Type '{type}' is already registered under resource '{resource}'." + ) + + self._resource_registry[resource][type] = resource_class + + def create(self, resource: str, type: str, *args: Any, **kwargs: Any) -> T: + """ + Create an instance of the class associated with the given resource and type. + """ + if resource not in self._resource_registry: + raise ValueError(f"Resource '{resource}' is not registered.") + + if type not in self._resource_registry[resource]: + raise ValueError( + f"Type '{type}' is not registered under resource '{resource}'." + ) + + cls = self._resource_registry[resource][type] + return cls(*args, **kwargs) diff --git a/pkgs/swarmauri/swarmauri/factories/concrete/__init__.py b/pkgs/swarmauri/swarmauri/factories/concrete/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py b/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py new file mode 100644 index 000000000..ee1f8b0f1 --- /dev/null +++ b/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py @@ -0,0 +1,35 @@ +import pytest +from swarmauri.factories.concrete.AgentFactory import AgentFactory + +class MockParser: + def __init__(self, name: str): + self.name = name + + +def test_agent_factory_register_and_create(): + factory = AgentFactory() + + # Register a resource + factory.register("MockParser", MockParser) + + # Create an instance + instance = factory.create("MockParser", name="TestParser") + assert isinstance(instance, MockParser) + assert instance.name == "TestParser" + + +def test_agent_factory_duplicate_register(): + factory = AgentFactory() + factory.register("MockParser", MockParser) + + # Attempt to register the same type again + with pytest.raises(ValueError, match="Type 'MockParser' is already registered."): + factory.register("MockParser", MockParser) + + +def test_agent_factory_create_unregistered_type(): + factory = AgentFactory() + + # Attempt to create an unregistered type + with pytest.raises(ValueError, match="Type 'UnregisteredType' is not registered."): + factory.create("UnregisteredType") diff --git a/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py b/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py new file mode 100644 index 000000000..cc9657bab --- /dev/null +++ b/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py @@ -0,0 +1,50 @@ +import pytest +from swarmauri.factories.concrete.Factory import Factory +from swarmauri.parsers.concrete.BeautifulSoupElementParser import BeautifulSoupElementParser + +def test_factory_register_and_create(): + factory = Factory() + + # Register a resource and type + factory.register("Parser", "BeautifulSoupElementParser", BeautifulSoupElementParser) + + html_content = "

Sample HTML content

" + + # Create an instance + instance = factory.create("Parser", "BeautifulSoupElementParser", element=html_content) + assert isinstance(instance, BeautifulSoupElementParser) + assert instance.type == "BeautifulSoupElementParser" + + +def test_factory_duplicate_register(): + factory = Factory() + factory.register("Parser", "BeautifulSoupElementParser", BeautifulSoupElementParser) + + # Attempt to register the same type again + with pytest.raises( + ValueError, + match="Type 'BeautifulSoupElementParser' is already registered under resource 'Parser'.", + ): + factory.register("Parser", "BeautifulSoupElementParser", BeautifulSoupElementParser) + + +def test_factory_create_unregistered_resource(): + factory = Factory() + + # Attempt to create an instance of an unregistered resource + with pytest.raises( + ValueError, match="Resource 'UnknownResource' is not registered." + ): + factory.create("UnknownResource", "BeautifulSoupElementParser") + + +def test_factory_create_unregistered_type(): + factory = Factory() + factory.register("Parser", "BeautifulSoupElementParser", BeautifulSoupElementParser) + + # Attempt to create an instance of an unregistered type + with pytest.raises( + ValueError, + match="Type 'UnknownType' is not registered under resource 'Parser'.", + ): + factory.create("Parser", "UnknownType") From 9cbe94501dffba509392c9455a9841704f541c79 Mon Sep 17 00:00:00 2001 From: michaeldecent2 <111002205+MichaelDecent@users.noreply.github.com> Date: Tue, 10 Dec 2024 23:16:01 +0100 Subject: [PATCH 050/168] swarm feat: enhance factory pattern implementation; add Factory resource type and update AgentFactory and Factory classes --- pkgs/core/swarmauri_core/ComponentBase.py | 1 + .../swarmauri/factories/base/FactoryBase.py | 19 ++++-- .../factories/concrete/AgentFactory.py | 8 +-- .../swarmauri/factories/concrete/Factory.py | 11 ++-- .../unit/factories/AgentFactory_unit_test.py | 65 ++++++++++++++----- .../tests/unit/factories/Factory_unit_test.py | 1 + 6 files changed, 72 insertions(+), 33 deletions(-) diff --git a/pkgs/core/swarmauri_core/ComponentBase.py b/pkgs/core/swarmauri_core/ComponentBase.py index a86c8b4fb..a5e558380 100644 --- a/pkgs/core/swarmauri_core/ComponentBase.py +++ b/pkgs/core/swarmauri_core/ComponentBase.py @@ -57,6 +57,7 @@ class ResourceTypes(Enum): VECTOR = "Vector" VCM = "VCM" DATA_CONNECTOR = "DataConnector" + FACTORY = "Factory" def generate_id() -> str: diff --git a/pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py b/pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py index c4748c543..1836180ae 100644 --- a/pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py +++ b/pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py @@ -1,20 +1,31 @@ -from typing import Any, Type +from typing import Any, Literal, Optional, Type +from swarmauri_core.ComponentBase import ComponentBase +from swarmauri_core.ComponentBase import ResourceTypes from swarmauri_core.factories.IFactory import IFactory, T +from pydantic import ConfigDict, Field -class FactoryBase(IFactory[T]): +class FactoryBase(ComponentBase, IFactory[T]): """ Base factory class for registering and creating instances. """ + resource: Optional[str] = Field(default=ResourceTypes.FACTORY.value, frozen=True) + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + type: Literal["FactoryBase"] = "FactoryBase" + def register(self, resource: str, type: str, resource_class: Type[T]) -> None: """ Register a resource class under a specific resource and type. """ - raise NotImplementedError("register method must be implemented in derived classes.") + raise NotImplementedError( + "register method must be implemented in derived classes." + ) def create(self, resource: str, type: str, *args: Any, **kwargs: Any) -> T: """ Create an instance of the class associated with the given resource and type. """ - raise NotImplementedError("create method must be implemented in derived classes.") \ No newline at end of file + raise NotImplementedError( + "create method must be implemented in derived classes." + ) diff --git a/pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py b/pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py index 16f513870..99b648300 100644 --- a/pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py +++ b/pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Type +from typing import Any, Dict, Literal, Type from swarmauri.factories.base.FactoryBase import FactoryBase, T @@ -6,10 +6,8 @@ class AgentFactory(FactoryBase[T]): """ Class-specific factory for managing resources and types. """ - - def __init__(self) -> None: - # Make _registry an instance-level variable. - self._registry: Dict[str, Type[T]] = {} + type: Literal['AgentFactory'] = 'AgentFactory' + _registry: Dict[str, Type[T]] = {} def register(self, type: str, resource_class: Type[T]) -> None: """ diff --git a/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py b/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py index 1be46451b..df278a9ee 100644 --- a/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py +++ b/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Type +from typing import Any, Dict, Literal, Type from swarmauri.factories.base.FactoryBase import FactoryBase, T @@ -6,11 +6,10 @@ class Factory(FactoryBase[T]): """ Non-recursive factory extending FactoryBase. """ - def __init__(self) -> None: - # Make _resource_registry an instance-level variable. - self._resource_registry: Dict[str, Dict[str, Type[T]]] = {} + type: Literal['Factory'] = 'Factory' + _resource_registry: Dict[str, Dict[str, Type[T]]] = {} - def register(self, resource: str, type: str, resource_class: Type[T]) -> None: + def register(self, resource: str, type: str) -> None: """ Register a resource class under a specific resource and type. """ @@ -22,7 +21,7 @@ def register(self, resource: str, type: str, resource_class: Type[T]) -> None: f"Type '{type}' is already registered under resource '{resource}'." ) - self._resource_registry[resource][type] = resource_class + self._resource_registry[resource][type] = eval(type) def create(self, resource: str, type: str, *args: Any, **kwargs: Any) -> T: """ diff --git a/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py b/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py index ee1f8b0f1..4c7289493 100644 --- a/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py +++ b/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py @@ -1,35 +1,64 @@ import pytest from swarmauri.factories.concrete.AgentFactory import AgentFactory +import os +from swarmauri.llms.concrete.GroqModel import GroqModel +from swarmauri.agents.concrete import QAAgent +from dotenv import load_dotenv -class MockParser: - def __init__(self, name: str): - self.name = name +load_dotenv() -def test_agent_factory_register_and_create(): - factory = AgentFactory() +@pytest.fixture(scope="module") +def groq_model(): + API_KEY = os.getenv("GROQ_API_KEY") + if not API_KEY: + pytest.skip("Skipping due to environment variable not set") + llm = GroqModel(api_key=API_KEY) + return llm - # Register a resource - factory.register("MockParser", MockParser) + +@pytest.fixture(scope="module") +def agent_factory(): + return AgentFactory() + + +@pytest.mark.unit +def test_ubc_resource(agent_factory): + assert agent_factory.resource == "Factory" + + +@pytest.mark.unit +def test_ubc_type(agent_factory): + assert agent_factory.type == "AgentFactory" + + +@pytest.mark.unit +def test_serialization(agent_factory): + assert ( + agent_factory.id + == AgentFactory.model_validate_json(agent_factory.model_dump_json()).id + ) + + +def test_agent_factory_register_and_create(agent_factory, groq_model): + + agent_factory.register(type="QAAgent", resource_class=QAAgent) # Create an instance - instance = factory.create("MockParser", name="TestParser") - assert isinstance(instance, MockParser) - assert instance.name == "TestParser" + instance = agent_factory.create(type="QAAgent", llm=groq_model) + assert isinstance(instance, QAAgent) + assert instance.type == "QAAgent" -def test_agent_factory_duplicate_register(): - factory = AgentFactory() - factory.register("MockParser", MockParser) +def test_agent_factory_duplicate_register(agent_factory): # Attempt to register the same type again - with pytest.raises(ValueError, match="Type 'MockParser' is already registered."): - factory.register("MockParser", MockParser) + with pytest.raises(ValueError, match="Type 'QAAgent' is already registered."): + agent_factory.register(type="QAAgent", resource_class=QAAgent) -def test_agent_factory_create_unregistered_type(): - factory = AgentFactory() +def test_agent_factory_create_unregistered_type(agent_factory): # Attempt to create an unregistered type with pytest.raises(ValueError, match="Type 'UnregisteredType' is not registered."): - factory.create("UnregisteredType") + agent_factory.create(type="UnregisteredType") diff --git a/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py b/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py index cc9657bab..c0a81d97f 100644 --- a/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py +++ b/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py @@ -2,6 +2,7 @@ from swarmauri.factories.concrete.Factory import Factory from swarmauri.parsers.concrete.BeautifulSoupElementParser import BeautifulSoupElementParser + def test_factory_register_and_create(): factory = Factory() From 2766564c3e3c4bd3123417d7dfcab0386bde6a9f Mon Sep 17 00:00:00 2001 From: michaeldecent2 <111002205+MichaelDecent@users.noreply.github.com> Date: Tue, 10 Dec 2024 23:40:21 +0100 Subject: [PATCH 051/168] swarm - feat: update Factory registration method to accept resource class; enhance unit tests for Factory and AgentFactory --- .../swarmauri/factories/concrete/Factory.py | 4 +- .../unit/factories/AgentFactory_unit_test.py | 6 +-- .../tests/unit/factories/Factory_unit_test.py | 46 +++++++++++++------ 3 files changed, 38 insertions(+), 18 deletions(-) diff --git a/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py b/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py index df278a9ee..098fd4445 100644 --- a/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py +++ b/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py @@ -9,7 +9,7 @@ class Factory(FactoryBase[T]): type: Literal['Factory'] = 'Factory' _resource_registry: Dict[str, Dict[str, Type[T]]] = {} - def register(self, resource: str, type: str) -> None: + def register(self, resource: str, type: str, resource_class: Type[T]) -> None: """ Register a resource class under a specific resource and type. """ @@ -21,7 +21,7 @@ def register(self, resource: str, type: str) -> None: f"Type '{type}' is already registered under resource '{resource}'." ) - self._resource_registry[resource][type] = eval(type) + self._resource_registry[resource][type] = resource_class def create(self, resource: str, type: str, *args: Any, **kwargs: Any) -> T: """ diff --git a/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py b/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py index 4c7289493..9f973c27a 100644 --- a/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py +++ b/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py @@ -39,7 +39,7 @@ def test_serialization(agent_factory): == AgentFactory.model_validate_json(agent_factory.model_dump_json()).id ) - +@pytest.mark.unit def test_agent_factory_register_and_create(agent_factory, groq_model): agent_factory.register(type="QAAgent", resource_class=QAAgent) @@ -49,14 +49,14 @@ def test_agent_factory_register_and_create(agent_factory, groq_model): assert isinstance(instance, QAAgent) assert instance.type == "QAAgent" - +@pytest.mark.unit def test_agent_factory_duplicate_register(agent_factory): # Attempt to register the same type again with pytest.raises(ValueError, match="Type 'QAAgent' is already registered."): agent_factory.register(type="QAAgent", resource_class=QAAgent) - +@pytest.mark.unit def test_agent_factory_create_unregistered_type(agent_factory): # Attempt to create an unregistered type diff --git a/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py b/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py index c0a81d97f..14934e75f 100644 --- a/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py +++ b/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py @@ -1,10 +1,31 @@ import pytest from swarmauri.factories.concrete.Factory import Factory -from swarmauri.parsers.concrete.BeautifulSoupElementParser import BeautifulSoupElementParser +from swarmauri.parsers.concrete.BeautifulSoupElementParser import ( + BeautifulSoupElementParser, +) -def test_factory_register_and_create(): - factory = Factory() +@pytest.fixture(scope="module") +def factory(): + return Factory() + + +@pytest.mark.unit +def test_ubc_resource(factory): + assert factory.resource == "Factory" + + +@pytest.mark.unit +def test_ubc_type(factory): + assert factory.type == "Factory" + + +@pytest.mark.unit +def test_serialization(factory): + assert factory.id == Factory.model_validate_json(factory.model_dump_json()).id + + +def test_factory_register_and_create(factory): # Register a resource and type factory.register("Parser", "BeautifulSoupElementParser", BeautifulSoupElementParser) @@ -12,25 +33,26 @@ def test_factory_register_and_create(): html_content = "

Sample HTML content

" # Create an instance - instance = factory.create("Parser", "BeautifulSoupElementParser", element=html_content) + instance = factory.create( + "Parser", "BeautifulSoupElementParser", element=html_content + ) assert isinstance(instance, BeautifulSoupElementParser) assert instance.type == "BeautifulSoupElementParser" -def test_factory_duplicate_register(): - factory = Factory() - factory.register("Parser", "BeautifulSoupElementParser", BeautifulSoupElementParser) +def test_factory_duplicate_register(factory): # Attempt to register the same type again with pytest.raises( ValueError, match="Type 'BeautifulSoupElementParser' is already registered under resource 'Parser'.", ): - factory.register("Parser", "BeautifulSoupElementParser", BeautifulSoupElementParser) + factory.register( + "Parser", "BeautifulSoupElementParser", BeautifulSoupElementParser + ) -def test_factory_create_unregistered_resource(): - factory = Factory() +def test_factory_create_unregistered_resource(factory): # Attempt to create an instance of an unregistered resource with pytest.raises( @@ -39,9 +61,7 @@ def test_factory_create_unregistered_resource(): factory.create("UnknownResource", "BeautifulSoupElementParser") -def test_factory_create_unregistered_type(): - factory = Factory() - factory.register("Parser", "BeautifulSoupElementParser", BeautifulSoupElementParser) +def test_factory_create_unregistered_type(factory): # Attempt to create an instance of an unregistered type with pytest.raises( From 358faf59239398bcd4c2851c64d598053083d975 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 10 Dec 2024 22:55:49 -0600 Subject: [PATCH 052/168] Delete pkgs/core/swarmauri_core/agent_factories/IAgentFactory.py --- .../agent_factories/IAgentFactory.py | 77 ------------------- 1 file changed, 77 deletions(-) delete mode 100644 pkgs/core/swarmauri_core/agent_factories/IAgentFactory.py diff --git a/pkgs/core/swarmauri_core/agent_factories/IAgentFactory.py b/pkgs/core/swarmauri_core/agent_factories/IAgentFactory.py deleted file mode 100644 index bbd75b2ce..000000000 --- a/pkgs/core/swarmauri_core/agent_factories/IAgentFactory.py +++ /dev/null @@ -1,77 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Type, Any -from datetime import datetime - -class IAgentFactory(ABC): - """ - Interface for Agent Factories, extended to include properties like ID, name, type, - creation date, and last modification date. - """ - - @abstractmethod - def create_agent(self, agent_type: str, **kwargs) -> Any: - pass - - @abstractmethod - def register_agent(self, agent_type: str, constructor: Type[Any]) -> None: - pass - - # Abstract properties and setters - @property - @abstractmethod - def id(self) -> str: - """Unique identifier for the factory instance.""" - pass - - @id.setter - @abstractmethod - def id(self, value: str) -> None: - pass - - @property - @abstractmethod - def name(self) -> str: - """Name of the factory.""" - pass - - @name.setter - @abstractmethod - def name(self, value: str) -> None: - pass - - @property - @abstractmethod - def type(self) -> str: - """Type of agents this factory produces.""" - pass - - @type.setter - @abstractmethod - def type(self, value: str) -> None: - pass - - @property - @abstractmethod - def date_created(self) -> datetime: - """The creation date of the factory instance.""" - pass - - @property - @abstractmethod - def last_modified(self) -> datetime: - """Date when the factory was last modified.""" - pass - - @last_modified.setter - @abstractmethod - def last_modified(self, value: datetime) -> None: - pass - - def __hash__(self): - """ - The __hash__ method allows objects of this class to be used in sets and as dictionary keys. - __hash__ should return an integer and be defined based on immutable properties. - This is generally implemented directly in concrete classes rather than in the interface, - but it's declared here to indicate that implementing classes must provide it. - """ - pass From d38ada41b0f711e042a126476a0d9362a0300625 Mon Sep 17 00:00:00 2001 From: michaeldecent2 <111002205+MichaelDecent@users.noreply.github.com> Date: Wed, 11 Dec 2024 11:07:57 +0100 Subject: [PATCH 053/168] feat: refactor Factory registration method to use dynamic class retrieval; add utility for module class import --- .../swarmauri/factories/concrete/Factory.py | 17 ++++----- .../swarmauri/utils/_get_subclasses.py | 35 +++++++++++++++++++ .../unit/factories/AgentFactory_unit_test.py | 3 ++ .../tests/unit/factories/Factory_unit_test.py | 13 ++++--- 4 files changed, 53 insertions(+), 15 deletions(-) create mode 100644 pkgs/swarmauri/swarmauri/utils/_get_subclasses.py diff --git a/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py b/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py index 098fd4445..be5261e87 100644 --- a/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py +++ b/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py @@ -1,5 +1,7 @@ +import logging from typing import Any, Dict, Literal, Type from swarmauri.factories.base.FactoryBase import FactoryBase, T +from swarmauri.utils._get_subclasses import get_classes_from_module class Factory(FactoryBase[T]): @@ -9,19 +11,14 @@ class Factory(FactoryBase[T]): type: Literal['Factory'] = 'Factory' _resource_registry: Dict[str, Dict[str, Type[T]]] = {} - def register(self, resource: str, type: str, resource_class: Type[T]) -> None: + def register(self, resource: str) -> None: """ - Register a resource class under a specific resource and type. + Register a resource class under a specific resource """ if resource not in self._resource_registry: - self._resource_registry[resource] = {} - - if type in self._resource_registry[resource]: - raise ValueError( - f"Type '{type}' is already registered under resource '{resource}'." - ) - - self._resource_registry[resource][type] = resource_class + self._resource_registry[resource] = get_classes_from_module(resource) + else: + raise ValueError(f"Resource '{resource}' is already registered.") def create(self, resource: str, type: str, *args: Any, **kwargs: Any) -> T: """ diff --git a/pkgs/swarmauri/swarmauri/utils/_get_subclasses.py b/pkgs/swarmauri/swarmauri/utils/_get_subclasses.py new file mode 100644 index 000000000..3ef299947 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/utils/_get_subclasses.py @@ -0,0 +1,35 @@ +import importlib + + +def get_classes_from_module(module_name: str): + """ + Dynamically imports a module and retrieves a dictionary of class names and their corresponding class objects. + + :param module_name: The name of the module (e.g., "parsers", "agent"). + :return: A dictionary with class names as keys and class objects as values. + """ + # Convert module name to lowercase to ensure consistency + module_name_lower = module_name.lower() + + # Construct the full module path dynamically + full_module_path = f"swarmauri.{module_name_lower}s.concrete" + + try: + # Import the module dynamically + module = importlib.import_module(full_module_path) + + # Get the list of class names from __all__ + class_names = getattr(module, "__all__", []) + + # Create a dictionary with class names and their corresponding class objects + classes_dict = { + class_name: getattr(module, class_name) for class_name in class_names + } + + return classes_dict + except ImportError as e: + print(f"Error importing module {full_module_path}: {e}") + raise e + except AttributeError as e: + print(f"Error accessing class in {full_module_path}: {e}") + raise e diff --git a/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py b/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py index 9f973c27a..c1eedcb6b 100644 --- a/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py +++ b/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py @@ -39,6 +39,7 @@ def test_serialization(agent_factory): == AgentFactory.model_validate_json(agent_factory.model_dump_json()).id ) + @pytest.mark.unit def test_agent_factory_register_and_create(agent_factory, groq_model): @@ -49,6 +50,7 @@ def test_agent_factory_register_and_create(agent_factory, groq_model): assert isinstance(instance, QAAgent) assert instance.type == "QAAgent" + @pytest.mark.unit def test_agent_factory_duplicate_register(agent_factory): @@ -56,6 +58,7 @@ def test_agent_factory_duplicate_register(agent_factory): with pytest.raises(ValueError, match="Type 'QAAgent' is already registered."): agent_factory.register(type="QAAgent", resource_class=QAAgent) + @pytest.mark.unit def test_agent_factory_create_unregistered_type(agent_factory): diff --git a/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py b/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py index 14934e75f..11a2e4f4f 100644 --- a/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py +++ b/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py @@ -3,6 +3,7 @@ from swarmauri.parsers.concrete.BeautifulSoupElementParser import ( BeautifulSoupElementParser, ) +from swarmauri.parsers.base.ParserBase import ParserBase @pytest.fixture(scope="module") @@ -25,10 +26,11 @@ def test_serialization(factory): assert factory.id == Factory.model_validate_json(factory.model_dump_json()).id +@pytest.mark.unit def test_factory_register_and_create(factory): # Register a resource and type - factory.register("Parser", "BeautifulSoupElementParser", BeautifulSoupElementParser) + factory.register("Parser") html_content = "

Sample HTML content

" @@ -40,18 +42,18 @@ def test_factory_register_and_create(factory): assert instance.type == "BeautifulSoupElementParser" +@pytest.mark.unit def test_factory_duplicate_register(factory): # Attempt to register the same type again with pytest.raises( ValueError, - match="Type 'BeautifulSoupElementParser' is already registered under resource 'Parser'.", + match="Resource 'Parser' is already registered.", ): - factory.register( - "Parser", "BeautifulSoupElementParser", BeautifulSoupElementParser - ) + factory.register("Parser") +@pytest.mark.unit def test_factory_create_unregistered_resource(factory): # Attempt to create an instance of an unregistered resource @@ -61,6 +63,7 @@ def test_factory_create_unregistered_resource(factory): factory.create("UnknownResource", "BeautifulSoupElementParser") +@pytest.mark.unit def test_factory_create_unregistered_type(factory): # Attempt to create an instance of an unregistered type From 321d22b2841e6570800438e4c3451c784c14e506 Mon Sep 17 00:00:00 2001 From: 3rdSon Date: Wed, 11 Dec 2024 11:49:06 +0100 Subject: [PATCH 054/168] addded state to resource type --- pkgs/core/swarmauri_core/ComponentBase.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkgs/core/swarmauri_core/ComponentBase.py b/pkgs/core/swarmauri_core/ComponentBase.py index 13a096001..6ac7c7dba 100644 --- a/pkgs/core/swarmauri_core/ComponentBase.py +++ b/pkgs/core/swarmauri_core/ComponentBase.py @@ -89,9 +89,6 @@ def __swm_register_subclass__(cls, subclass) -> None: f"Subclass {subclass.__name__} does not have a type annotation" ) - # [subclass.__swm_reset_class__() for subclass in cls.__swm_subclasses__ - # if hasattr(subclass, '__swm_reset_class__')] - @classmethod def __swm_reset_class__(cls): logging.debug("__swm_reset_class__ executed\n") @@ -124,9 +121,6 @@ def __swm_reset_class__(cls): cls.__annotations__[each] = sc cls.__fields__[each].annotation = sc - # This is not necessary as the model_rebuild address forward_refs - # https://docs.pydantic.dev/latest/api/base_model/#pydantic.BaseModel.model_post_init - # cls.update_forward_refs() cls.model_rebuild(force=True) @field_validator("type") From 87aa38f64462767dc2b06f3411f0a6c419bdc3e2 Mon Sep 17 00:00:00 2001 From: michaeldecent2 <111002205+MichaelDecent@users.noreply.github.com> Date: Wed, 11 Dec 2024 12:54:12 +0100 Subject: [PATCH 055/168] feat: simplify IFactory and FactoryBase interfaces by removing generic type parameters; update create and register methods to accept Any type --- pkgs/core/swarmauri_core/factories/IFactory.py | 13 +++++-------- .../swarmauri/factories/base/FactoryBase.py | 8 ++++---- .../swarmauri/factories/concrete/AgentFactory.py | 12 ++++++------ .../swarmauri/factories/concrete/Factory.py | 11 +++++------ .../tests/unit/factories/AgentFactory_unit_test.py | 2 ++ .../tests/unit/factories/Factory_unit_test.py | 1 - 6 files changed, 22 insertions(+), 25 deletions(-) diff --git a/pkgs/core/swarmauri_core/factories/IFactory.py b/pkgs/core/swarmauri_core/factories/IFactory.py index eed40b31b..9421339ed 100644 --- a/pkgs/core/swarmauri_core/factories/IFactory.py +++ b/pkgs/core/swarmauri_core/factories/IFactory.py @@ -1,21 +1,18 @@ from abc import ABC, abstractmethod -from typing import Any, Generic, Type, TypeVar +from typing import Any -# Generic type variable for flexible factory implementations -T = TypeVar("T") - -class IFactory(ABC, Generic[T]): +class IFactory(ABC): """ Interface defining core methods for factories. """ @abstractmethod - def create(self, *args: Any, **kwargs: Any) -> T: - """Create and return an instance of type T.""" + def create(self, *args: Any, **kwargs: Any) -> Any: + """Create and return an instance.""" pass @abstractmethod - def register(self, resource: str, name: str, resource_class: Type[T]) -> None: + def register(self, resource: str, name: str) -> None: """Register a class with the factory.""" pass diff --git a/pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py b/pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py index 1836180ae..cf223d7ed 100644 --- a/pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py +++ b/pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py @@ -1,11 +1,11 @@ from typing import Any, Literal, Optional, Type from swarmauri_core.ComponentBase import ComponentBase from swarmauri_core.ComponentBase import ResourceTypes -from swarmauri_core.factories.IFactory import IFactory, T +from swarmauri_core.factories.IFactory import IFactory from pydantic import ConfigDict, Field -class FactoryBase(ComponentBase, IFactory[T]): +class FactoryBase(ComponentBase, IFactory): """ Base factory class for registering and creating instances. """ @@ -14,7 +14,7 @@ class FactoryBase(ComponentBase, IFactory[T]): model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) type: Literal["FactoryBase"] = "FactoryBase" - def register(self, resource: str, type: str, resource_class: Type[T]) -> None: + def register(self, resource: str, type: str) -> None: """ Register a resource class under a specific resource and type. """ @@ -22,7 +22,7 @@ def register(self, resource: str, type: str, resource_class: Type[T]) -> None: "register method must be implemented in derived classes." ) - def create(self, resource: str, type: str, *args: Any, **kwargs: Any) -> T: + def create(self, resource: str, type: str, *args: Any, **kwargs: Any) -> Any: """ Create an instance of the class associated with the given resource and type. """ diff --git a/pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py b/pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py index 99b648300..23991fe88 100644 --- a/pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py +++ b/pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py @@ -1,15 +1,15 @@ -from typing import Any, Dict, Literal, Type -from swarmauri.factories.base.FactoryBase import FactoryBase, T +from typing import Any, Callable, Dict, Literal +from swarmauri.factories.base.FactoryBase import FactoryBase -class AgentFactory(FactoryBase[T]): +class AgentFactory(FactoryBase): """ Class-specific factory for managing resources and types. """ type: Literal['AgentFactory'] = 'AgentFactory' - _registry: Dict[str, Type[T]] = {} + _registry: Dict[str, Callable] = {} - def register(self, type: str, resource_class: Type[T]) -> None: + def register(self, type: str, resource_class: Callable) -> None: """ Register a resource class with a specific type. """ @@ -17,7 +17,7 @@ def register(self, type: str, resource_class: Type[T]) -> None: raise ValueError(f"Type '{type}' is already registered.") self._registry[type] = resource_class - def create(self, type: str, *args: Any, **kwargs: Any) -> T: + def create(self, type: str, *args: Any, **kwargs: Any) -> Any: """ Create an instance of the class associated with the given type name. """ diff --git a/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py b/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py index be5261e87..3dade4998 100644 --- a/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py +++ b/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py @@ -1,15 +1,14 @@ -import logging -from typing import Any, Dict, Literal, Type -from swarmauri.factories.base.FactoryBase import FactoryBase, T +from typing import Any, Callable, Dict, Literal +from swarmauri.factories.base.FactoryBase import FactoryBase from swarmauri.utils._get_subclasses import get_classes_from_module -class Factory(FactoryBase[T]): +class Factory(FactoryBase): """ Non-recursive factory extending FactoryBase. """ type: Literal['Factory'] = 'Factory' - _resource_registry: Dict[str, Dict[str, Type[T]]] = {} + _resource_registry: Dict[str, Dict[str, Callable]] = {} def register(self, resource: str) -> None: """ @@ -20,7 +19,7 @@ def register(self, resource: str) -> None: else: raise ValueError(f"Resource '{resource}' is already registered.") - def create(self, resource: str, type: str, *args: Any, **kwargs: Any) -> T: + def create(self, resource: str, type: str, *args: Any, **kwargs: Any) -> Any: """ Create an instance of the class associated with the given resource and type. """ diff --git a/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py b/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py index c1eedcb6b..30ca0867e 100644 --- a/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py +++ b/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py @@ -1,3 +1,4 @@ +import logging import pytest from swarmauri.factories.concrete.AgentFactory import AgentFactory import os @@ -49,6 +50,7 @@ def test_agent_factory_register_and_create(agent_factory, groq_model): instance = agent_factory.create(type="QAAgent", llm=groq_model) assert isinstance(instance, QAAgent) assert instance.type == "QAAgent" + logging.info(instance.resource) @pytest.mark.unit diff --git a/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py b/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py index 11a2e4f4f..4e1ad4a33 100644 --- a/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py +++ b/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py @@ -3,7 +3,6 @@ from swarmauri.parsers.concrete.BeautifulSoupElementParser import ( BeautifulSoupElementParser, ) -from swarmauri.parsers.base.ParserBase import ParserBase @pytest.fixture(scope="module") From d260e8201ec308605a1992580ed198171a6a3f1d Mon Sep 17 00:00:00 2001 From: michaeldecent2 <111002205+MichaelDecent@users.noreply.github.com> Date: Wed, 11 Dec 2024 13:25:08 +0100 Subject: [PATCH 056/168] swarm - Ensuring everything aligns --- pkgs/core/swarmauri_core/factories/IFactory.py | 6 +++--- pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py | 6 +++--- pkgs/swarmauri/swarmauri/factories/concrete/Factory.py | 5 +++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pkgs/core/swarmauri_core/factories/IFactory.py b/pkgs/core/swarmauri_core/factories/IFactory.py index 9421339ed..a30fc7d15 100644 --- a/pkgs/core/swarmauri_core/factories/IFactory.py +++ b/pkgs/core/swarmauri_core/factories/IFactory.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Any +from typing import Any, Callable class IFactory(ABC): @@ -8,11 +8,11 @@ class IFactory(ABC): """ @abstractmethod - def create(self, *args: Any, **kwargs: Any) -> Any: + def create(self, type: str, *args: Any, **kwargs: Any) -> Any: """Create and return an instance.""" pass @abstractmethod - def register(self, resource: str, name: str) -> None: + def register(self, type: str, resource_class: Callable) -> None: """Register a class with the factory.""" pass diff --git a/pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py b/pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py index cf223d7ed..a52776bf9 100644 --- a/pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py +++ b/pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py @@ -1,4 +1,4 @@ -from typing import Any, Literal, Optional, Type +from typing import Any, Callable, Literal, Optional, Type from swarmauri_core.ComponentBase import ComponentBase from swarmauri_core.ComponentBase import ResourceTypes from swarmauri_core.factories.IFactory import IFactory @@ -14,7 +14,7 @@ class FactoryBase(ComponentBase, IFactory): model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) type: Literal["FactoryBase"] = "FactoryBase" - def register(self, resource: str, type: str) -> None: + def register(self, type: str, resource_class: Callable) -> None: """ Register a resource class under a specific resource and type. """ @@ -22,7 +22,7 @@ def register(self, resource: str, type: str) -> None: "register method must be implemented in derived classes." ) - def create(self, resource: str, type: str, *args: Any, **kwargs: Any) -> Any: + def create(self, type: str, *args: Any, **kwargs: Any) -> Any: """ Create an instance of the class associated with the given resource and type. """ diff --git a/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py b/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py index 3dade4998..9ea48fe69 100644 --- a/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py +++ b/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py @@ -7,12 +7,13 @@ class Factory(FactoryBase): """ Non-recursive factory extending FactoryBase. """ - type: Literal['Factory'] = 'Factory' + + type: Literal["Factory"] = "Factory" _resource_registry: Dict[str, Dict[str, Callable]] = {} def register(self, resource: str) -> None: """ - Register a resource class under a specific resource + Register a resource class under a specific resource. """ if resource not in self._resource_registry: self._resource_registry[resource] = get_classes_from_module(resource) From ec023dab88b7acdc15960a4814169f5a95ace533 Mon Sep 17 00:00:00 2001 From: 3rdSon Date: Wed, 11 Dec 2024 15:25:39 +0100 Subject: [PATCH 057/168] Base and core transport --- .../swarmauri_core/transport/ITransport.py | 42 ++++++++++++++ .../core/swarmauri_core/transport/__init__.py | 42 ++++++++++++++ .../swarmauri/swarmauri/transport/__init__.py | 1 + .../swarmauri/transport/base/TransportBase.py | 55 +++++++++++++++++++ .../swarmauri/transport/base/__init__.py | 0 .../swarmauri/transport/concrete/__init__.py | 0 6 files changed, 140 insertions(+) create mode 100644 pkgs/core/swarmauri_core/transport/ITransport.py create mode 100644 pkgs/core/swarmauri_core/transport/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/transport/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/transport/base/TransportBase.py create mode 100644 pkgs/swarmauri/swarmauri/transport/base/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/transport/concrete/__init__.py diff --git a/pkgs/core/swarmauri_core/transport/ITransport.py b/pkgs/core/swarmauri_core/transport/ITransport.py new file mode 100644 index 000000000..0d621efbb --- /dev/null +++ b/pkgs/core/swarmauri_core/transport/ITransport.py @@ -0,0 +1,42 @@ +from abc import ABC, abstractmethod +from typing import Any, List + + +class ITransportComm(ABC): + """ + Interface defining standard transportation methods for agent interactions + """ + @abstractmethod + def send(self, sender: str, recipient: str, message: Any) -> None: + """ + Send a message to a specific recipient + """ + pass + + @abstractmethod + def broadcast(self, sender: str, message: Any) -> None: + """ + Broadcast a message to all agents + """ + pass + + @abstractmethod + def multicast(self, sender: str, recipients: List[str], message: Any) -> None: + """ + Send a message to multiple specific recipients + """ + pass + + @abstractmethod + def subscribe(self, topic: str, subscriber: str) -> None: + """ + Subscribe to a specific transportation topic + """ + pass + + @abstractmethod + def publish(self, topic: str, message: Any) -> None: + """ + Publish a message to a specific topic + """ + pass \ No newline at end of file diff --git a/pkgs/core/swarmauri_core/transport/__init__.py b/pkgs/core/swarmauri_core/transport/__init__.py new file mode 100644 index 000000000..4cb714616 --- /dev/null +++ b/pkgs/core/swarmauri_core/transport/__init__.py @@ -0,0 +1,42 @@ +from abc import ABC, abstractmethod +from typing import Any, List + + +class ITransportComm(ABC): + """ + Interface defining standard communication methods for agent interactions + """ + @abstractmethod + def send(self, sender: str, recipient: str, message: Any) -> None: + """ + Send a message to a specific recipient + """ + pass + + @abstractmethod + def broadcast(self, sender: str, message: Any) -> None: + """ + Broadcast a message to all agents + """ + pass + + @abstractmethod + def multicast(self, sender: str, recipients: List[str], message: Any) -> None: + """ + Send a message to multiple specific recipients + """ + pass + + @abstractmethod + def subscribe(self, topic: str, subscriber: str) -> None: + """ + Subscribe to a specific communication topic + """ + pass + + @abstractmethod + def publish(self, topic: str, message: Any) -> None: + """ + Publish a message to a specific topic + """ + pass \ No newline at end of file diff --git a/pkgs/swarmauri/swarmauri/transport/__init__.py b/pkgs/swarmauri/swarmauri/transport/__init__.py new file mode 100644 index 000000000..f3ada0e57 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/transport/__init__.py @@ -0,0 +1 @@ +from swarmauri.transport.concrete import * diff --git a/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py b/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py new file mode 100644 index 000000000..1d6d35781 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py @@ -0,0 +1,55 @@ +from typing import Dict, Any, List +from enum import Enum, auto +import datetime +from swarmauri_core.transport.ITransport import ITransport + +class TransportationProtocol(Enum): + """ + Enumeration of transportation protocols supported by the transport layer + """ + UNICAST = auto() + MULTICAST = auto() + BROADCAST = auto() + PUBSUB = auto() + +class TransportBase(ITransport): + """ + Base class for transport transportation with shared utilities + """ + def __init__(self, name: str): + self.name = name + self.subscriptions: Dict[str, List[str]] = {} + self.message_history: List[Dict[str, Any]] = [] + + def log_message(self, sender: str, recipients: List[str], message: Any, protocol: TransportationProtocol): + """ + Log transportation events + """ + log_entry = { + 'sender': sender, + 'recipients': recipients, + 'message': message, + 'protocol': protocol, + 'timestamp': datetime.now() + } + self.message_history.append(log_entry) + + def send(self, sender: str, recipient: str, message: Any) -> None: + raise NotImplementedError("Subclasses must implement send method") + + def broadcast(self, sender: str, message: Any) -> None: + raise NotImplementedError("Subclasses must implement broadcast method") + + def multicast(self, sender: str, recipients: List[str], message: Any) -> None: + raise NotImplementedError("Subclasses must implement multicast method") + + def subscribe(self, topic: str, subscriber: str) -> None: + if topic not in self.subscriptions: + self.subscriptions[topic] = [] + if subscriber not in self.subscriptions[topic]: + self.subscriptions[topic].append(subscriber) + + def publish(self, topic: str, message: Any) -> None: + if topic in self.subscriptions: + for subscriber in self.subscriptions[topic]: + self.send(topic, subscriber, message) \ No newline at end of file diff --git a/pkgs/swarmauri/swarmauri/transport/base/__init__.py b/pkgs/swarmauri/swarmauri/transport/base/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/transport/concrete/__init__.py b/pkgs/swarmauri/swarmauri/transport/concrete/__init__.py new file mode 100644 index 000000000..e69de29bb From cff2a163a686f304378e8433c736d36af9f79b9a Mon Sep 17 00:00:00 2001 From: michaeldecent2 <111002205+MichaelDecent@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:10:26 +0100 Subject: [PATCH 058/168] feat: refactor Factory and AgentFactory classes; update registration methods and enhance error handling; add utility for dynamic class retrieval --- .../core/swarmauri_core/factories/IFactory.py | 10 ++--- .../swarmauri/factories/base/FactoryBase.py | 2 +- .../factories/concrete/AgentFactory.py | 22 +++++++++-- .../swarmauri/factories/concrete/Factory.py | 6 +-- .../swarmauri/utils/_get_subclasses.py | 37 +++++++++++++++++++ .../unit/factories/AgentFactory_unit_test.py | 19 ++++------ .../tests/unit/factories/Factory_unit_test.py | 20 ++-------- 7 files changed, 74 insertions(+), 42 deletions(-) diff --git a/pkgs/core/swarmauri_core/factories/IFactory.py b/pkgs/core/swarmauri_core/factories/IFactory.py index a30fc7d15..f751e19ee 100644 --- a/pkgs/core/swarmauri_core/factories/IFactory.py +++ b/pkgs/core/swarmauri_core/factories/IFactory.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Any, Callable +from typing import Any class IFactory(ABC): @@ -12,7 +12,7 @@ def create(self, type: str, *args: Any, **kwargs: Any) -> Any: """Create and return an instance.""" pass - @abstractmethod - def register(self, type: str, resource_class: Callable) -> None: - """Register a class with the factory.""" - pass + # @abstractmethod + # def register(self, type: str, resource_class: Callable) -> None: + # """Register a class with the factory.""" + # pass diff --git a/pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py b/pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py index a52776bf9..73218afdb 100644 --- a/pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py +++ b/pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py @@ -1,4 +1,4 @@ -from typing import Any, Callable, Literal, Optional, Type +from typing import Any, Callable, Literal, Optional from swarmauri_core.ComponentBase import ComponentBase from swarmauri_core.ComponentBase import ResourceTypes from swarmauri_core.factories.IFactory import IFactory diff --git a/pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py b/pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py index 23991fe88..26c6e47ef 100644 --- a/pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py +++ b/pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py @@ -1,28 +1,42 @@ from typing import Any, Callable, Dict, Literal from swarmauri.factories.base.FactoryBase import FactoryBase +from swarmauri.utils._get_subclasses import get_class_from_module class AgentFactory(FactoryBase): """ Class-specific factory for managing resources and types. """ - type: Literal['AgentFactory'] = 'AgentFactory' + + type: Literal["AgentFactory"] = "AgentFactory" _registry: Dict[str, Callable] = {} - def register(self, type: str, resource_class: Callable) -> None: + def _register(self, type: str, resource: str = "Agent") -> None: """ Register a resource class with a specific type. """ if type in self._registry: raise ValueError(f"Type '{type}' is already registered.") - self._registry[type] = resource_class + resource_cls = get_class_from_module(resource, type) + if resource_cls is not None: + self._registry[type] = resource_cls + else: + raise ValueError(f"Type '{type}' is not found in resource '{resource}'.") def create(self, type: str, *args: Any, **kwargs: Any) -> Any: """ Create an instance of the class associated with the given type name. """ + self._register(type) + if type not in self._registry: - raise ValueError(f"Type '{type}' is not registered.") + raise ValueError(f"Type '{type}' is not found.") cls = self._registry[type] return cls(*args, **kwargs) + + def get_agents(self): + """ + Return a list of registered agent types. + """ + return list(self._registry.keys()) diff --git a/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py b/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py index 9ea48fe69..5e0cdabbb 100644 --- a/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py +++ b/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py @@ -11,7 +11,7 @@ class Factory(FactoryBase): type: Literal["Factory"] = "Factory" _resource_registry: Dict[str, Dict[str, Callable]] = {} - def register(self, resource: str) -> None: + def _register(self, resource: str) -> None: """ Register a resource class under a specific resource. """ @@ -25,11 +25,11 @@ def create(self, resource: str, type: str, *args: Any, **kwargs: Any) -> Any: Create an instance of the class associated with the given resource and type. """ if resource not in self._resource_registry: - raise ValueError(f"Resource '{resource}' is not registered.") + self._register(resource) if type not in self._resource_registry[resource]: raise ValueError( - f"Type '{type}' is not registered under resource '{resource}'." + f"Type '{type}' does not exist under resource '{resource}'." ) cls = self._resource_registry[resource][type] diff --git a/pkgs/swarmauri/swarmauri/utils/_get_subclasses.py b/pkgs/swarmauri/swarmauri/utils/_get_subclasses.py index 3ef299947..a0e9162a4 100644 --- a/pkgs/swarmauri/swarmauri/utils/_get_subclasses.py +++ b/pkgs/swarmauri/swarmauri/utils/_get_subclasses.py @@ -1,4 +1,5 @@ import importlib +import re def get_classes_from_module(module_name: str): @@ -29,7 +30,43 @@ def get_classes_from_module(module_name: str): return classes_dict except ImportError as e: print(f"Error importing module {full_module_path}: {e}") + raise ModuleNotFoundError(f"Resource '{module_name}' is not found.") + except AttributeError as e: + print(f"Error accessing class in {full_module_path}: {e}") raise e + + +def get_class_from_module(module_name: str, class_name: str): + """ + Dynamically imports a module and retrieves the class name of the module. + + :param module_name: The name of the module (e.g., "parsers", "agent"). + :return: The class name of the module. + """ + # Convert module name to lowercase to ensure consistency + module_name_lower = module_name.lower() + + # Construct the full module path dynamically + full_module_path = f"swarmauri.{module_name_lower}s.concrete" + + try: + # Import the module dynamically + module = importlib.import_module(full_module_path) + + # Get the list of class names from __all__ + class_names = getattr(module, "__all__", []) + + if not class_names: + raise AttributeError(f"No classes found in module {full_module_path}") + + for cls_name in class_names: + if cls_name == class_name: + return getattr(module, class_name) + return None + + except ImportError as e: + print(f"Error importing module {full_module_path}: {e}") + raise ModuleNotFoundError(f"Resource '{module_name}' is not found.") except AttributeError as e: print(f"Error accessing class in {full_module_path}: {e}") raise e diff --git a/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py b/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py index 30ca0867e..000b8e8ac 100644 --- a/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py +++ b/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py @@ -1,4 +1,3 @@ -import logging import pytest from swarmauri.factories.concrete.AgentFactory import AgentFactory import os @@ -44,26 +43,22 @@ def test_serialization(agent_factory): @pytest.mark.unit def test_agent_factory_register_and_create(agent_factory, groq_model): - agent_factory.register(type="QAAgent", resource_class=QAAgent) - # Create an instance instance = agent_factory.create(type="QAAgent", llm=groq_model) assert isinstance(instance, QAAgent) assert instance.type == "QAAgent" - logging.info(instance.resource) @pytest.mark.unit -def test_agent_factory_duplicate_register(agent_factory): +def test_agent_factory_create_unregistered_type(agent_factory): - # Attempt to register the same type again - with pytest.raises(ValueError, match="Type 'QAAgent' is already registered."): - agent_factory.register(type="QAAgent", resource_class=QAAgent) + # Attempt to create an unregistered type + with pytest.raises(ValueError, match="Type 'UnregisteredType' is not found."): + agent_factory.create(type="UnregisteredType") @pytest.mark.unit -def test_agent_factory_create_unregistered_type(agent_factory): +def test_agent_factory_get_agents(agent_factory): - # Attempt to create an unregistered type - with pytest.raises(ValueError, match="Type 'UnregisteredType' is not registered."): - agent_factory.create(type="UnregisteredType") + assert agent_factory.get_agents() == ["QAAgent"] + assert len(agent_factory.get_agents()) == 1 diff --git a/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py b/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py index 4e1ad4a33..1b161c725 100644 --- a/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py +++ b/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py @@ -26,10 +26,7 @@ def test_serialization(factory): @pytest.mark.unit -def test_factory_register_and_create(factory): - - # Register a resource and type - factory.register("Parser") +def test_factory_create(factory): html_content = "

Sample HTML content

" @@ -41,23 +38,12 @@ def test_factory_register_and_create(factory): assert instance.type == "BeautifulSoupElementParser" -@pytest.mark.unit -def test_factory_duplicate_register(factory): - - # Attempt to register the same type again - with pytest.raises( - ValueError, - match="Resource 'Parser' is already registered.", - ): - factory.register("Parser") - - @pytest.mark.unit def test_factory_create_unregistered_resource(factory): # Attempt to create an instance of an unregistered resource with pytest.raises( - ValueError, match="Resource 'UnknownResource' is not registered." + ModuleNotFoundError, match="Resource 'UnknownResource' is not found." ): factory.create("UnknownResource", "BeautifulSoupElementParser") @@ -68,6 +54,6 @@ def test_factory_create_unregistered_type(factory): # Attempt to create an instance of an unregistered type with pytest.raises( ValueError, - match="Type 'UnknownType' is not registered under resource 'Parser'.", + match="Type 'UnknownType' does not exist under resource 'Parser'.", ): factory.create("Parser", "UnknownType") From 1bf50de384be8cca87f1863ca94e6dba2d45c34c Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Thu, 12 Dec 2024 09:53:06 +0100 Subject: [PATCH 059/168] feat: update IFactory and Factory classes to enhance registration methods; --- .../core/swarmauri_core/factories/IFactory.py | 10 ++++----- .../factories/concrete/AgentFactory.py | 12 +++------- .../swarmauri/factories/concrete/Factory.py | 16 +++++++++----- .../unit/factories/AgentFactory_unit_test.py | 4 +++- .../tests/unit/factories/Factory_unit_test.py | 22 ++++++++++++++++--- 5 files changed, 41 insertions(+), 23 deletions(-) diff --git a/pkgs/core/swarmauri_core/factories/IFactory.py b/pkgs/core/swarmauri_core/factories/IFactory.py index f751e19ee..a30fc7d15 100644 --- a/pkgs/core/swarmauri_core/factories/IFactory.py +++ b/pkgs/core/swarmauri_core/factories/IFactory.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Any +from typing import Any, Callable class IFactory(ABC): @@ -12,7 +12,7 @@ def create(self, type: str, *args: Any, **kwargs: Any) -> Any: """Create and return an instance.""" pass - # @abstractmethod - # def register(self, type: str, resource_class: Callable) -> None: - # """Register a class with the factory.""" - # pass + @abstractmethod + def register(self, type: str, resource_class: Callable) -> None: + """Register a class with the factory.""" + pass diff --git a/pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py b/pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py index 26c6e47ef..aa15146ba 100644 --- a/pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py +++ b/pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py @@ -11,26 +11,20 @@ class AgentFactory(FactoryBase): type: Literal["AgentFactory"] = "AgentFactory" _registry: Dict[str, Callable] = {} - def _register(self, type: str, resource: str = "Agent") -> None: + def register(self, type: str, resource_class: Callable) -> None: """ Register a resource class with a specific type. """ if type in self._registry: raise ValueError(f"Type '{type}' is already registered.") - resource_cls = get_class_from_module(resource, type) - if resource_cls is not None: - self._registry[type] = resource_cls - else: - raise ValueError(f"Type '{type}' is not found in resource '{resource}'.") + self._registry[type] = resource_class def create(self, type: str, *args: Any, **kwargs: Any) -> Any: """ Create an instance of the class associated with the given type name. """ - self._register(type) - if type not in self._registry: - raise ValueError(f"Type '{type}' is not found.") + raise ValueError(f"Type '{type}' is not registered.") cls = self._registry[type] return cls(*args, **kwargs) diff --git a/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py b/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py index 5e0cdabbb..4540e2c42 100644 --- a/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py +++ b/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py @@ -11,25 +11,31 @@ class Factory(FactoryBase): type: Literal["Factory"] = "Factory" _resource_registry: Dict[str, Dict[str, Callable]] = {} - def _register(self, resource: str) -> None: + def register(self, resource: str, type: str, resource_class: Callable) -> None: """ Register a resource class under a specific resource. """ + if type in self._resource_registry.get(resource, {}): + raise ValueError( + f"Type '{type}' is already registered under resource '{resource}'." + ) + if resource not in self._resource_registry: self._resource_registry[resource] = get_classes_from_module(resource) - else: - raise ValueError(f"Resource '{resource}' is already registered.") + + if type not in self._resource_registry[resource]: + self._resource_registry[resource][type] = resource_class def create(self, resource: str, type: str, *args: Any, **kwargs: Any) -> Any: """ Create an instance of the class associated with the given resource and type. """ if resource not in self._resource_registry: - self._register(resource) + raise ValueError(f"Resource '{resource}' is not registered.") if type not in self._resource_registry[resource]: raise ValueError( - f"Type '{type}' does not exist under resource '{resource}'." + f"Type '{type}' is not registered under resource '{resource}'." ) cls = self._resource_registry[resource][type] diff --git a/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py b/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py index 000b8e8ac..5a0254b70 100644 --- a/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py +++ b/pkgs/swarmauri/tests/unit/factories/AgentFactory_unit_test.py @@ -43,6 +43,8 @@ def test_serialization(agent_factory): @pytest.mark.unit def test_agent_factory_register_and_create(agent_factory, groq_model): + agent_factory.register(type="QAAgent", resource_class=QAAgent) + # Create an instance instance = agent_factory.create(type="QAAgent", llm=groq_model) assert isinstance(instance, QAAgent) @@ -53,7 +55,7 @@ def test_agent_factory_register_and_create(agent_factory, groq_model): def test_agent_factory_create_unregistered_type(agent_factory): # Attempt to create an unregistered type - with pytest.raises(ValueError, match="Type 'UnregisteredType' is not found."): + with pytest.raises(ValueError, match="Type 'UnregisteredType' is not registered."): agent_factory.create(type="UnregisteredType") diff --git a/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py b/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py index 1b161c725..92434909f 100644 --- a/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py +++ b/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py @@ -26,7 +26,10 @@ def test_serialization(factory): @pytest.mark.unit -def test_factory_create(factory): +def test_factory_register_create_resource(factory): + + # Register a resource and type + factory.register("Parser", "BeautifulSoupElementParser", BeautifulSoupElementParser) html_content = "

Sample HTML content

" @@ -43,17 +46,30 @@ def test_factory_create_unregistered_resource(factory): # Attempt to create an instance of an unregistered resource with pytest.raises( - ModuleNotFoundError, match="Resource 'UnknownResource' is not found." + ValueError, match="Resource 'UnknownResource' is not registered." ): factory.create("UnknownResource", "BeautifulSoupElementParser") +@pytest.mark.unit +def test_factory_duplicate_register(factory): + + # Attempt to register the same type again + with pytest.raises( + ValueError, + match="Type 'BeautifulSoupElementParser' is already registered under resource 'Parser'.", + ): + factory.register( + "Parser", "BeautifulSoupElementParser", BeautifulSoupElementParser + ) + + @pytest.mark.unit def test_factory_create_unregistered_type(factory): # Attempt to create an instance of an unregistered type with pytest.raises( ValueError, - match="Type 'UnknownType' does not exist under resource 'Parser'.", + match="Type 'UnknownType' is not registered under resource 'Parser'.", ): factory.create("Parser", "UnknownType") From 2f0937d58e339c8ae61e2b1cd6277b76c6b6baeb Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 12 Dec 2024 03:09:17 -0600 Subject: [PATCH 060/168] Update FactoryBase.py --- pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py b/pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py index 73218afdb..ef1b9324e 100644 --- a/pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py +++ b/pkgs/swarmauri/swarmauri/factories/base/FactoryBase.py @@ -5,7 +5,7 @@ from pydantic import ConfigDict, Field -class FactoryBase(ComponentBase, IFactory): +class FactoryBase(IFactory, ComponentBase): """ Base factory class for registering and creating instances. """ From 5a0dcb2a0c9e3f77897fd950ef086118472bb699 Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Thu, 12 Dec 2024 10:37:29 +0100 Subject: [PATCH 061/168] feat: implement lazy loading for Factory and AgentFactory classes in concrete module --- .../swarmauri/factories/concrete/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkgs/swarmauri/swarmauri/factories/concrete/__init__.py b/pkgs/swarmauri/swarmauri/factories/concrete/__init__.py index e69de29bb..014b941f2 100644 --- a/pkgs/swarmauri/swarmauri/factories/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/factories/concrete/__init__.py @@ -0,0 +1,14 @@ +from swarmauri.utils._lazy_import import _lazy_import + +# List of swarms names (file names without the ".py" extension) and corresponding class names +factories_files = [ + ("swarmauri.factories.concrete.Factory", "Factory"), + ("swarmauri.factories.concrete.AgentFactory", "AgentFactory"), +] + +# Lazy loading of swarms classes, storing them in variables +for module_name, class_name in factories_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded swarms classes to __all__ +__all__ = [class_name for _, class_name in factories_files] From 51776571bef4a20f532b0ec89fccc74a6d4aea28 Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Thu, 12 Dec 2024 11:01:59 +0100 Subject: [PATCH 062/168] fix: update error handling for unregistered resources in Factory --- pkgs/swarmauri/swarmauri/factories/concrete/Factory.py | 2 +- pkgs/swarmauri/swarmauri/utils/_get_subclasses.py | 2 +- pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py b/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py index 4540e2c42..9242ffd5d 100644 --- a/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py +++ b/pkgs/swarmauri/swarmauri/factories/concrete/Factory.py @@ -31,7 +31,7 @@ def create(self, resource: str, type: str, *args: Any, **kwargs: Any) -> Any: Create an instance of the class associated with the given resource and type. """ if resource not in self._resource_registry: - raise ValueError(f"Resource '{resource}' is not registered.") + self._resource_registry[resource] = get_classes_from_module(resource) if type not in self._resource_registry[resource]: raise ValueError( diff --git a/pkgs/swarmauri/swarmauri/utils/_get_subclasses.py b/pkgs/swarmauri/swarmauri/utils/_get_subclasses.py index a0e9162a4..adf100601 100644 --- a/pkgs/swarmauri/swarmauri/utils/_get_subclasses.py +++ b/pkgs/swarmauri/swarmauri/utils/_get_subclasses.py @@ -30,7 +30,7 @@ def get_classes_from_module(module_name: str): return classes_dict except ImportError as e: print(f"Error importing module {full_module_path}: {e}") - raise ModuleNotFoundError(f"Resource '{module_name}' is not found.") + raise ModuleNotFoundError(f"Resource '{module_name}' is not registered.") except AttributeError as e: print(f"Error accessing class in {full_module_path}: {e}") raise e diff --git a/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py b/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py index 92434909f..11695f7ec 100644 --- a/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py +++ b/pkgs/swarmauri/tests/unit/factories/Factory_unit_test.py @@ -46,7 +46,7 @@ def test_factory_create_unregistered_resource(factory): # Attempt to create an instance of an unregistered resource with pytest.raises( - ValueError, match="Resource 'UnknownResource' is not registered." + ModuleNotFoundError, match="Resource 'UnknownResource' is not registered." ): factory.create("UnknownResource", "BeautifulSoupElementParser") From 35aace63956afab9d9f787e86734f024d5c4c1a3 Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Thu, 12 Dec 2024 11:03:31 +0100 Subject: [PATCH 063/168] minor fix --- pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py b/pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py index aa15146ba..e641bf8f1 100644 --- a/pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py +++ b/pkgs/swarmauri/swarmauri/factories/concrete/AgentFactory.py @@ -1,6 +1,5 @@ from typing import Any, Callable, Dict, Literal from swarmauri.factories.base.FactoryBase import FactoryBase -from swarmauri.utils._get_subclasses import get_class_from_module class AgentFactory(FactoryBase): From 46f655d85c38b6ff8fb7281ac0f3711b4aae9baf Mon Sep 17 00:00:00 2001 From: 3rdSon Date: Thu, 12 Dec 2024 13:41:39 +0100 Subject: [PATCH 064/168] corrected the base class --- pkgs/core/swarmauri_core/ComponentBase.py | 1 + .../swarmauri_core/transport/ITransport.py | 17 +---- .../core/swarmauri_core/transport/__init__.py | 42 ------------ .../swarmauri/transport/base/TransportBase.py | 65 +++++++++---------- .../transport/concrete/PubSubTransport.py | 0 .../swarmauri/transport/concrete/__init__.py | 13 ++++ .../transport/PubSubTransport_unit_test.py | 0 7 files changed, 45 insertions(+), 93 deletions(-) create mode 100644 pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py create mode 100644 pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py diff --git a/pkgs/core/swarmauri_core/ComponentBase.py b/pkgs/core/swarmauri_core/ComponentBase.py index 13a096001..e6f3a4d5c 100644 --- a/pkgs/core/swarmauri_core/ComponentBase.py +++ b/pkgs/core/swarmauri_core/ComponentBase.py @@ -53,6 +53,7 @@ class ResourceTypes(Enum): VECTOR = "Vector" VCM = "VCM" DATA_CONNECTOR = "DataConnector" + TRANSPORT = "Transport" def generate_id() -> str: diff --git a/pkgs/core/swarmauri_core/transport/ITransport.py b/pkgs/core/swarmauri_core/transport/ITransport.py index 0d621efbb..c9f811473 100644 --- a/pkgs/core/swarmauri_core/transport/ITransport.py +++ b/pkgs/core/swarmauri_core/transport/ITransport.py @@ -2,10 +2,11 @@ from typing import Any, List -class ITransportComm(ABC): +class ITransport(ABC): """ Interface defining standard transportation methods for agent interactions """ + @abstractmethod def send(self, sender: str, recipient: str, message: Any) -> None: """ @@ -26,17 +27,3 @@ def multicast(self, sender: str, recipients: List[str], message: Any) -> None: Send a message to multiple specific recipients """ pass - - @abstractmethod - def subscribe(self, topic: str, subscriber: str) -> None: - """ - Subscribe to a specific transportation topic - """ - pass - - @abstractmethod - def publish(self, topic: str, message: Any) -> None: - """ - Publish a message to a specific topic - """ - pass \ No newline at end of file diff --git a/pkgs/core/swarmauri_core/transport/__init__.py b/pkgs/core/swarmauri_core/transport/__init__.py index 4cb714616..e69de29bb 100644 --- a/pkgs/core/swarmauri_core/transport/__init__.py +++ b/pkgs/core/swarmauri_core/transport/__init__.py @@ -1,42 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Any, List - - -class ITransportComm(ABC): - """ - Interface defining standard communication methods for agent interactions - """ - @abstractmethod - def send(self, sender: str, recipient: str, message: Any) -> None: - """ - Send a message to a specific recipient - """ - pass - - @abstractmethod - def broadcast(self, sender: str, message: Any) -> None: - """ - Broadcast a message to all agents - """ - pass - - @abstractmethod - def multicast(self, sender: str, recipients: List[str], message: Any) -> None: - """ - Send a message to multiple specific recipients - """ - pass - - @abstractmethod - def subscribe(self, topic: str, subscriber: str) -> None: - """ - Subscribe to a specific communication topic - """ - pass - - @abstractmethod - def publish(self, topic: str, message: Any) -> None: - """ - Publish a message to a specific topic - """ - pass \ No newline at end of file diff --git a/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py b/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py index 1d6d35781..2020f677f 100644 --- a/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py +++ b/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py @@ -1,55 +1,48 @@ -from typing import Dict, Any, List +from typing import Dict, Any, List, Optional, Literal from enum import Enum, auto -import datetime +from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes from swarmauri_core.transport.ITransport import ITransport + class TransportationProtocol(Enum): """ Enumeration of transportation protocols supported by the transport layer """ + UNICAST = auto() MULTICAST = auto() BROADCAST = auto() PUBSUB = auto() -class TransportBase(ITransport): - """ - Base class for transport transportation with shared utilities - """ - def __init__(self, name: str): - self.name = name - self.subscriptions: Dict[str, List[str]] = {} - self.message_history: List[Dict[str, Any]] = [] - def log_message(self, sender: str, recipients: List[str], message: Any, protocol: TransportationProtocol): - """ - Log transportation events - """ - log_entry = { - 'sender': sender, - 'recipients': recipients, - 'message': message, - 'protocol': protocol, - 'timestamp': datetime.now() - } - self.message_history.append(log_entry) +class TransportBase(ITransport, ComponentBase): + allowed_protocols: List[TransportationProtocol] = [] + resource: Optional[str] = ResourceTypes.TRANSPORT.value + type: Literal["TransportBase"] = "TransportBase" def send(self, sender: str, recipient: str, message: Any) -> None: - raise NotImplementedError("Subclasses must implement send method") + """ + Send a message to a specific recipient. + + Raises: + NotImplementedError: Subclasses must implement this method. + """ + raise NotImplementedError("send() not implemented in subclass yet.") def broadcast(self, sender: str, message: Any) -> None: - raise NotImplementedError("Subclasses must implement broadcast method") + """ + Broadcast a message to all potential recipients. + + Raises: + NotImplementedError: Subclasses must implement this method. + """ + raise NotImplementedError("broadcast() not implemented in subclass yet.") def multicast(self, sender: str, recipients: List[str], message: Any) -> None: - raise NotImplementedError("Subclasses must implement multicast method") - - def subscribe(self, topic: str, subscriber: str) -> None: - if topic not in self.subscriptions: - self.subscriptions[topic] = [] - if subscriber not in self.subscriptions[topic]: - self.subscriptions[topic].append(subscriber) - - def publish(self, topic: str, message: Any) -> None: - if topic in self.subscriptions: - for subscriber in self.subscriptions[topic]: - self.send(topic, subscriber, message) \ No newline at end of file + """ + Send a message to multiple specific recipients. + + Raises: + NotImplementedError: Subclasses must implement this method. + """ + raise NotImplementedError("multicast() not implemented in subclass yet.") diff --git a/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py b/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/transport/concrete/__init__.py b/pkgs/swarmauri/swarmauri/transport/concrete/__init__.py index e69de29bb..893b547b4 100644 --- a/pkgs/swarmauri/swarmauri/transport/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/transport/concrete/__init__.py @@ -0,0 +1,13 @@ +from swarmauri.utils._lazy_import import _lazy_import + +# List of transport names (file names without the ".py" extension) and corresponding class names +transport_files = [ + ("swarmauri.transport.concrete.PubSubTransport", "PubSubTransport"), +] + +# Lazy loading of transport classes, storing them in variables +for module_name, class_name in transport_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded transport classes to __all__ +__all__ = [class_name for _, class_name in transport_files] diff --git a/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py b/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py new file mode 100644 index 000000000..e69de29bb From e46db2df86e8b09118b581994222eedac3beac3d Mon Sep 17 00:00:00 2001 From: 3rdSon Date: Thu, 12 Dec 2024 14:24:26 +0100 Subject: [PATCH 065/168] corrected the merge issue --- pkgs/core/swarmauri_core/ComponentBase.py | 70 ++++++++++++++++++++++- 1 file changed, 67 insertions(+), 3 deletions(-) diff --git a/pkgs/core/swarmauri_core/ComponentBase.py b/pkgs/core/swarmauri_core/ComponentBase.py index e6f3a4d5c..63b3f6ea8 100644 --- a/pkgs/core/swarmauri_core/ComponentBase.py +++ b/pkgs/core/swarmauri_core/ComponentBase.py @@ -1,12 +1,13 @@ +import json from typing import ( + Any, + Dict, Optional, List, Literal, TypeVar, Type, Union, - Annotated, - Generic, ClassVar, Set, get_args, @@ -16,11 +17,14 @@ from enum import Enum import inspect import hashlib -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field, ValidationError, field_validator import logging from swarmauri_core.typing import SubclassUnion +T = TypeVar("T", bound="ComponentBase") + + class ResourceTypes(Enum): UNIVERSAL_BASE = "ComponentBase" AGENT = "Agent" @@ -54,6 +58,7 @@ class ResourceTypes(Enum): VCM = "VCM" DATA_CONNECTOR = "DataConnector" TRANSPORT = "Transport" + FACTORY = "Factory" def generate_id() -> str: @@ -181,3 +186,62 @@ def swm_path(self): @property def swm_isremote(self): return bool(self.host) + + @classmethod + def model_validate_json( + cls: Type[T], json_payload: Union[str, Dict[str, Any]], strict: bool = False + ) -> T: + # Ensure we're working with a dictionary + if isinstance(json_payload, str): + try: + payload_dict = json.loads(json_payload) + except json.JSONDecodeError: + raise ValueError("Invalid JSON payload") + else: + payload_dict = json_payload + + # Try to determine the specific component type + component_type = payload_dict.get("type", "ComponentBase") + + # Attempt to find the correct subclass + target_cls = cls.get_subclass_by_type(component_type) + + # Fallback logic + if target_cls is None: + if strict: + raise ValueError(f"Cannot resolve component type: {component_type}") + target_cls = cls + logging.warning( + f"Falling back to base ComponentBase for type: {component_type}" + ) + + # Validate using the determined class + try: + return target_cls.model_validate(payload_dict) + except ValidationError as e: + logging.error(f"Validation failed for {component_type}: {e}") + raise + + @classmethod + def get_subclass_by_type(cls, type_name: str) -> Optional[Type["ComponentBase"]]: + # First, check for exact match in registered subclasses + for subclass in cls.__swm_subclasses__: + if ( + subclass.__name__ == type_name + or getattr(subclass, "type", None) == type_name + ): + return subclass + + # If no exact match, try case-insensitive search + for subclass in cls.__swm_subclasses__: + if ( + subclass.__name__.lower() == type_name.lower() + or str(getattr(subclass, "type", "")).lower() == type_name.lower() + ): + return subclass + + return None + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "ComponentBase": + return cls.model_validate(data) From 04ec1427ff997b6c4a042839f52d286d4f950e4b Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Thu, 12 Dec 2024 14:40:47 +0100 Subject: [PATCH 066/168] swarm - feat: added service registry --- pkgs/core/swarmauri_core/ComponentBase.py | 1 + .../service_registries/IServiceRegistry.py | 29 ++++++++++++ .../service_registries/__init__.py | 0 .../swarmauri/service_registries/__init__.py | 0 .../base/ServiceRegistryBase.py | 40 ++++++++++++++++ .../service_registries/base/__init__.py | 0 .../concrete/ServiceRegistry.py | 10 ++++ .../service_registries/concrete/__init__.py | 0 .../ServiceRegistry_unit_test.py | 46 +++++++++++++++++++ 9 files changed, 126 insertions(+) create mode 100644 pkgs/core/swarmauri_core/service_registries/IServiceRegistry.py create mode 100644 pkgs/core/swarmauri_core/service_registries/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/service_registries/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/service_registries/base/ServiceRegistryBase.py create mode 100644 pkgs/swarmauri/swarmauri/service_registries/base/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/service_registries/concrete/ServiceRegistry.py create mode 100644 pkgs/swarmauri/swarmauri/service_registries/concrete/__init__.py create mode 100644 pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py diff --git a/pkgs/core/swarmauri_core/ComponentBase.py b/pkgs/core/swarmauri_core/ComponentBase.py index a5e558380..4aad0dd25 100644 --- a/pkgs/core/swarmauri_core/ComponentBase.py +++ b/pkgs/core/swarmauri_core/ComponentBase.py @@ -58,6 +58,7 @@ class ResourceTypes(Enum): VCM = "VCM" DATA_CONNECTOR = "DataConnector" FACTORY = "Factory" + SERVICE_REGISTRY = "ServiceRegistry" def generate_id() -> str: diff --git a/pkgs/core/swarmauri_core/service_registries/IServiceRegistry.py b/pkgs/core/swarmauri_core/service_registries/IServiceRegistry.py new file mode 100644 index 000000000..e434514aa --- /dev/null +++ b/pkgs/core/swarmauri_core/service_registries/IServiceRegistry.py @@ -0,0 +1,29 @@ +from abc import ABC, abstractmethod +from typing import Dict, Any, List, Optional + + +class IServiceRegistry(ABC): + """ + Abstract base class for service registries. + """ + + @abstractmethod + def register_service(self, name: str, details: Dict[str, Any]) -> None: + """ + Register a new service with the given name and details. + """ + pass + + @abstractmethod + def get_service(self, name: str) -> Optional[Dict[str, Any]]: + """ + Retrieve a service by its name. + """ + pass + + @abstractmethod + def get_services_by_roles(self, roles: List[str]) -> List[str]: + """ + Get services filtered by their roles. + """ + pass diff --git a/pkgs/core/swarmauri_core/service_registries/__init__.py b/pkgs/core/swarmauri_core/service_registries/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/service_registries/__init__.py b/pkgs/swarmauri/swarmauri/service_registries/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/service_registries/base/ServiceRegistryBase.py b/pkgs/swarmauri/swarmauri/service_registries/base/ServiceRegistryBase.py new file mode 100644 index 000000000..ca6a8824d --- /dev/null +++ b/pkgs/swarmauri/swarmauri/service_registries/base/ServiceRegistryBase.py @@ -0,0 +1,40 @@ +from typing import Dict, Any, List, Literal, Optional + +from pydantic import ConfigDict, Field +from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes +from swarmauri_core.service_registries.IServiceRegistry import IServiceRegistry + + +class ServiceRegistryBase(IServiceRegistry, ComponentBase): + """ + Concrete implementation of the IServiceRegistry abstract base class. + """ + + services: Dict[str, Any] = {} + type: Literal["ServiceRegistryBase"] = "ServiceRegistryBase" + resource: Optional[str] = Field( + default=ResourceTypes.SERVICE_REGISTRY.value, frozen=True + ) + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + + def register_service(self, name: str, details: Dict[str, Any]) -> None: + """ + Register a new service with the given name and details. + """ + self.services[name] = details + + def get_service(self, name: str) -> Optional[Dict[str, Any]]: + """ + Retrieve a service by its name. + """ + return self.services.get(name) + + def get_services_by_roles(self, roles: List[str]) -> List[str]: + """ + Get services filtered by their roles. + """ + return [ + name + for name, details in self.services.items() + if details.get("role") in roles + ] diff --git a/pkgs/swarmauri/swarmauri/service_registries/base/__init__.py b/pkgs/swarmauri/swarmauri/service_registries/base/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/service_registries/concrete/ServiceRegistry.py b/pkgs/swarmauri/swarmauri/service_registries/concrete/ServiceRegistry.py new file mode 100644 index 000000000..d61d34452 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/service_registries/concrete/ServiceRegistry.py @@ -0,0 +1,10 @@ +from typing import Literal +from swarmauri.service_registries.base.ServiceRegistryBase import ServiceRegistryBase + + +class ServiceRegistry(ServiceRegistryBase): + """ + Concrete implementation of the ServiceRegistryBase. + """ + + type: Literal["ServiceRegistry"] = "ServiceRegistry" diff --git a/pkgs/swarmauri/swarmauri/service_registries/concrete/__init__.py b/pkgs/swarmauri/swarmauri/service_registries/concrete/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py b/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py new file mode 100644 index 000000000..fca37383a --- /dev/null +++ b/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py @@ -0,0 +1,46 @@ +import pytest +from swarmauri.service_registries.concrete.ServiceRegistry import ServiceRegistry + + +@pytest.fixture +def service_registry(): + return ServiceRegistry() + +@pytest.mark.unit +def test_ubc_resource(service_registry): + assert service_registry.resource == "ServiceRegistry" + + +@pytest.mark.unit +def test_ubc_type(service_registry): + assert service_registry.type == "ServiceRegistry" + + +@pytest.mark.unit +def test_serialization(service_registry): + assert service_registry.id == service_registry.model_validate_json(service_registry.model_dump_json()).id + + +def test_register_service(service_registry): + service_registry.register_service("auth", {"role": "authentication"}) + assert service_registry.services["auth"] == {"role": "authentication"} + + +def test_get_service(service_registry): + service_registry.register_service("auth", {"role": "authentication"}) + service = service_registry.get_service("auth") + assert service == {"role": "authentication"} + assert service_registry.get_service("nonexistent") is None + + +def test_get_services_by_roles(service_registry): + service_registry.register_service("auth", {"role": "authentication"}) + service_registry.register_service("db", {"role": "database"}) + recipients = service_registry.get_services_by_roles(["authentication"]) + assert recipients == ["auth"] + recipients = service_registry.get_services_by_roles(["database"]) + assert recipients == ["db"] + recipients = service_registry.get_services_by_roles( + ["authentication", "database"] + ) + assert set(recipients) == {"auth", "db"} From f031faa5b10ebbead7ca00f21b4d33b2f338ead2 Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Thu, 12 Dec 2024 14:46:28 +0100 Subject: [PATCH 067/168] test: improve formatting and add unit tests for ServiceRegistry --- .../service_registries/ServiceRegistry_unit_test.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py b/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py index fca37383a..9c1622a9d 100644 --- a/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py +++ b/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py @@ -6,6 +6,7 @@ def service_registry(): return ServiceRegistry() + @pytest.mark.unit def test_ubc_resource(service_registry): assert service_registry.resource == "ServiceRegistry" @@ -18,14 +19,19 @@ def test_ubc_type(service_registry): @pytest.mark.unit def test_serialization(service_registry): - assert service_registry.id == service_registry.model_validate_json(service_registry.model_dump_json()).id + assert ( + service_registry.id + == service_registry.model_validate_json(service_registry.model_dump_json()).id + ) +@pytest.mark.unit def test_register_service(service_registry): service_registry.register_service("auth", {"role": "authentication"}) assert service_registry.services["auth"] == {"role": "authentication"} +@pytest.mark.unit def test_get_service(service_registry): service_registry.register_service("auth", {"role": "authentication"}) service = service_registry.get_service("auth") @@ -33,6 +39,7 @@ def test_get_service(service_registry): assert service_registry.get_service("nonexistent") is None +@pytest.mark.unit def test_get_services_by_roles(service_registry): service_registry.register_service("auth", {"role": "authentication"}) service_registry.register_service("db", {"role": "database"}) @@ -40,7 +47,5 @@ def test_get_services_by_roles(service_registry): assert recipients == ["auth"] recipients = service_registry.get_services_by_roles(["database"]) assert recipients == ["db"] - recipients = service_registry.get_services_by_roles( - ["authentication", "database"] - ) + recipients = service_registry.get_services_by_roles(["authentication", "database"]) assert set(recipients) == {"auth", "db"} From e90a8576dde1d2b5f985c041dce3d35cc4c2fe50 Mon Sep 17 00:00:00 2001 From: 3rdSon Date: Thu, 12 Dec 2024 15:47:57 +0100 Subject: [PATCH 068/168] implemented pubsubtransport --- .../transport/concrete/PubSubTransport.py | 122 ++++++++++++++++++ .../transport/PubSubTransport_unit_test.py | 116 +++++++++++++++++ 2 files changed, 238 insertions(+) diff --git a/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py b/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py index e69de29bb..ccc07c221 100644 --- a/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py +++ b/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py @@ -0,0 +1,122 @@ +from typing import Dict, Any, List, Optional, Set +from uuid import uuid4 +import asyncio +from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes +from swarmauri_core.transport.ITransport import ITransport +from swarmauri.transport.base.TransportBase import TransportBase, TransportationProtocol + + +class PubSubTransport(TransportBase): + allowed_protocols: List[TransportationProtocol] = [TransportationProtocol.PUBSUB] + + def __init__(self): + """ + Initialize the Publish-Subscribe Broker. + Manages topics and subscriptions for agents. + """ + super().__init__() + self._topics: Dict[str, Set[str]] = {} # Topic to subscriber mappings + self._subscribers: Dict[str, asyncio.Queue] = ( + {} + ) # Subscriber ID to message queue + + async def subscribe(self, topic: str) -> str: + """ + Subscribe an agent to a specific topic. + + Args: + topic (str): The topic to subscribe to + + Returns: + str: Unique subscriber ID + """ + subscriber_id = str(uuid4()) + + # Create message queue for this subscriber + self._subscribers[subscriber_id] = asyncio.Queue() + + # Add subscriber to topic + if topic not in self._topics: + self._topics[topic] = set() + self._topics[topic].add(subscriber_id) + + return subscriber_id + + async def unsubscribe(self, topic: str, subscriber_id: str): + """ + Unsubscribe an agent from a topic. + + Args: + topic (str): The topic to unsubscribe from + subscriber_id (str): Unique identifier of the subscriber + """ + if topic in self._topics and subscriber_id in self._topics[topic]: + self._topics[topic].remove(subscriber_id) + + # Optional: Clean up if no subscribers remain + if not self._topics[topic]: + del self._topics[topic] + + async def publish(self, topic: str, message: Any): + """ + Publish a message to a specific topic. + + Args: + topic (str): The topic to publish to + message (Any): The message to be published + """ + if topic not in self._topics: + return + + # Distribute message to all subscribers of this topic + for subscriber_id in self._topics[topic]: + await self._subscribers[subscriber_id].put(message) + + async def receive(self, subscriber_id: str) -> Any: + """ + Receive messages for a specific subscriber. + + Args: + subscriber_id (str): Unique identifier of the subscriber + + Returns: + Any: Received message + """ + return await self._subscribers[subscriber_id].get() + + def send(self, sender: str, recipient: str, message: Any) -> None: + """ + Simulate sending a direct message (not applicable in Pub/Sub context). + + Args: + sender (str): The sender ID + recipient (str): The recipient ID + message (Any): The message to send + + Raises: + NotImplementedError: This method is not applicable for Pub/Sub. + """ + raise NotImplementedError("Direct send not supported in Pub/Sub model.") + + def broadcast(self, sender: str, message: Any) -> None: + """ + Broadcast a message to all subscribers of all topics. + + Args: + sender (str): The sender ID + message (Any): The message to broadcast + """ + for topic in self._topics: + asyncio.create_task(self.publish(topic, message)) + + def multicast(self, sender: str, recipients: List[str], message: Any) -> None: + """ + Send a message to specific topics (acting as recipients). + + Args: + sender (str): The sender ID + recipients (List[str]): Topics to send the message to + message (Any): The message to send + """ + for topic in recipients: + asyncio.create_task(self.publish(topic, message)) diff --git a/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py b/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py index e69de29bb..2d2c68f76 100644 --- a/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py +++ b/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py @@ -0,0 +1,116 @@ +import pytest +import asyncio +from uuid import UUID +from typing import Any +from swarmauri.transport.concrete.PubSubTransport import ( + PubSubTransport, +) +from swarmauri.utils.timeout_wrapper import timeout + + +@pytest.fixture +async def pubsub_transport(): + transport = PubSubTransport() + yield transport + + +@timeout(5) +@pytest.mark.unit +@pytest.mark.asyncio +async def test_subscribe(pubsub_transport): + topic = "test_topic" + subscriber_id = await pubsub_transport.subscribe(topic) + + # Validate subscriber ID format + assert isinstance(UUID(subscriber_id), UUID) + + # Ensure subscriber is added to the topic + assert subscriber_id in pubsub_transport._topics[topic] + + +@timeout(5) +@pytest.mark.unit +@pytest.mark.asyncio +async def test_unsubscribe(pubsub_transport): + topic = "test_topic" + subscriber_id = await pubsub_transport.subscribe(topic) + + await pubsub_transport.unsubscribe(topic, subscriber_id) + + # Ensure subscriber is removed from the topic + assert subscriber_id not in pubsub_transport._topics.get(topic, set()) + + +@timeout(5) +@pytest.mark.unit +@pytest.mark.asyncio +async def test_publish_and_receive(pubsub_transport): + topic = "test_topic" + subscriber_id = await pubsub_transport.subscribe(topic) + + message = "Hello, PubSub!" + await pubsub_transport.publish(topic, message) + + # Ensure the subscriber receives the message + received_message = await pubsub_transport.receive(subscriber_id) + assert received_message == message + + +@timeout(5) +@pytest.mark.unit +@pytest.mark.asyncio +async def test_broadcast(pubsub_transport): + topic1 = "topic1" + topic2 = "topic2" + subscriber_id1 = await pubsub_transport.subscribe(topic1) + subscriber_id2 = await pubsub_transport.subscribe(topic2) + + message = "Broadcast Message" + pubsub_transport.broadcast("sender_id", message) + + # Ensure both subscribers receive the message + received_message1 = await pubsub_transport.receive(subscriber_id1) + received_message2 = await pubsub_transport.receive(subscriber_id2) + assert received_message1 == message + assert received_message2 == message + + +@timeout(5) +@pytest.mark.unit +@pytest.mark.asyncio +async def test_multicast(pubsub_transport): + topic1 = "topic1" + topic2 = "topic2" + topic3 = "topic3" + subscriber_id1 = await pubsub_transport.subscribe(topic1) + subscriber_id2 = await pubsub_transport.subscribe(topic2) + subscriber_id3 = await pubsub_transport.subscribe(topic3) + + message = "Multicast Message" + pubsub_transport.multicast("sender_id", [topic1, topic2], message) + + # Ensure only subscribers of specified topics receive the message + received_message1 = await pubsub_transport.receive(subscriber_id1) + received_message2 = await pubsub_transport.receive(subscriber_id2) + assert received_message1 == message + assert received_message2 == message + + try: + await asyncio.wait_for(pubsub_transport.receive(subscriber_id3), timeout=1.0) + pytest.fail("Expected no message, but received one.") + except asyncio.TimeoutError: + pass + + +@timeout(5) +@pytest.mark.unit +@pytest.mark.asyncio +async def test_receive_no_messages(pubsub_transport): + topic = "test_topic" + subscriber_id = await pubsub_transport.subscribe(topic) + + try: + await asyncio.wait_for(pubsub_transport.receive(subscriber_id), timeout=1.0) + pytest.fail("Expected no message, but received one.") + except asyncio.TimeoutError: + pass From 8bf840bfdef78ad8422414ba765edc5b8aeaa698 Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Fri, 13 Dec 2024 08:38:11 +0100 Subject: [PATCH 069/168] feat: add deregister and update methods to IServiceRegistry and implement in ServiceRegistryBase --- .../service_registries/IServiceRegistry.py | 14 +++++++++ .../base/ServiceRegistryBase.py | 20 ++++++++++++ .../ServiceRegistry_unit_test.py | 31 +++++++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/pkgs/core/swarmauri_core/service_registries/IServiceRegistry.py b/pkgs/core/swarmauri_core/service_registries/IServiceRegistry.py index e434514aa..8549e52ef 100644 --- a/pkgs/core/swarmauri_core/service_registries/IServiceRegistry.py +++ b/pkgs/core/swarmauri_core/service_registries/IServiceRegistry.py @@ -27,3 +27,17 @@ def get_services_by_roles(self, roles: List[str]) -> List[str]: Get services filtered by their roles. """ pass + + @abstractmethod + def deregister_service(self, name: str) -> None: + """ + Deregister the service with the given name. + """ + pass + + @abstractmethod + def update_service(self, name: str, details: Dict[str, Any]) -> None: + """ + Update the details of the service with the given name. + """ + pass diff --git a/pkgs/swarmauri/swarmauri/service_registries/base/ServiceRegistryBase.py b/pkgs/swarmauri/swarmauri/service_registries/base/ServiceRegistryBase.py index ca6a8824d..6b0b33698 100644 --- a/pkgs/swarmauri/swarmauri/service_registries/base/ServiceRegistryBase.py +++ b/pkgs/swarmauri/swarmauri/service_registries/base/ServiceRegistryBase.py @@ -38,3 +38,23 @@ def get_services_by_roles(self, roles: List[str]) -> List[str]: for name, details in self.services.items() if details.get("role") in roles ] + + def deregister_service(self, name: str) -> None: + """ + Deregister the service with the given name. + """ + if name in self.services: + del self.services[name] + print(f"Service {name} deregistered.") + else: + raise ValueError(f"Service {name} not found.") + + def update_service(self, name: str, details: Dict[str, Any]) -> None: + """ + Update the details of the service with the given name. + """ + if name in self.services: + self.services[name].update(details) + print(f"Service {name} updated with new details: {details}") + else: + raise ValueError(f"Service {name} not found.") diff --git a/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py b/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py index 9c1622a9d..32f122fc6 100644 --- a/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py +++ b/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py @@ -49,3 +49,34 @@ def test_get_services_by_roles(service_registry): assert recipients == ["db"] recipients = service_registry.get_services_by_roles(["authentication", "database"]) assert set(recipients) == {"auth", "db"} + + +@pytest.mark.unit +def test_deregister_service(service_registry): + service_registry.register_service("auth", {"role": "authentication"}) + service_registry.deregister_service("auth") + assert "auth" not in service_registry.services + + +@pytest.mark.unit +def test_deregister_service_nonexistent(service_registry): + with pytest.raises(ValueError) as exc_info: + service_registry.deregister_service("nonexistent") + assert str(exc_info.value) == "Service nonexistent not found." + + +@pytest.mark.unit +def test_update_service(service_registry): + service_registry.register_service("auth", {"role": "authentication"}) + service_registry.update_service("auth", {"role": "auth_service", "version": "1.0"}) + assert service_registry.services["auth"] == { + "role": "auth_service", + "version": "1.0", + } + + +@pytest.mark.unit +def test_update_service_nonexistent(service_registry): + with pytest.raises(ValueError) as exc_info: + service_registry.update_service("nonexistent", {"role": "new_role"}) + assert str(exc_info.value) == "Service nonexistent not found." From 322380e144dd485da0abdcbfa41e7da008ce246e Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Fri, 13 Dec 2024 08:50:48 +0100 Subject: [PATCH 070/168] refactor: rename deregister_service to unregister_service --- .../swarmauri_core/service_registries/IServiceRegistry.py | 4 ++-- .../service_registries/base/ServiceRegistryBase.py | 6 +++--- .../unit/service_registries/ServiceRegistry_unit_test.py | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkgs/core/swarmauri_core/service_registries/IServiceRegistry.py b/pkgs/core/swarmauri_core/service_registries/IServiceRegistry.py index 8549e52ef..911dc7b19 100644 --- a/pkgs/core/swarmauri_core/service_registries/IServiceRegistry.py +++ b/pkgs/core/swarmauri_core/service_registries/IServiceRegistry.py @@ -29,9 +29,9 @@ def get_services_by_roles(self, roles: List[str]) -> List[str]: pass @abstractmethod - def deregister_service(self, name: str) -> None: + def unregister_service(self, name: str) -> None: """ - Deregister the service with the given name. + unregister the service with the given name. """ pass diff --git a/pkgs/swarmauri/swarmauri/service_registries/base/ServiceRegistryBase.py b/pkgs/swarmauri/swarmauri/service_registries/base/ServiceRegistryBase.py index 6b0b33698..a7e567a88 100644 --- a/pkgs/swarmauri/swarmauri/service_registries/base/ServiceRegistryBase.py +++ b/pkgs/swarmauri/swarmauri/service_registries/base/ServiceRegistryBase.py @@ -39,13 +39,13 @@ def get_services_by_roles(self, roles: List[str]) -> List[str]: if details.get("role") in roles ] - def deregister_service(self, name: str) -> None: + def unregister_service(self, name: str) -> None: """ - Deregister the service with the given name. + unregister the service with the given name. """ if name in self.services: del self.services[name] - print(f"Service {name} deregistered.") + print(f"Service {name} unregistered.") else: raise ValueError(f"Service {name} not found.") diff --git a/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py b/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py index 32f122fc6..6498d4c29 100644 --- a/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py +++ b/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py @@ -52,16 +52,16 @@ def test_get_services_by_roles(service_registry): @pytest.mark.unit -def test_deregister_service(service_registry): +def test_unregister_service(service_registry): service_registry.register_service("auth", {"role": "authentication"}) - service_registry.deregister_service("auth") + service_registry.unregister_service("auth") assert "auth" not in service_registry.services @pytest.mark.unit -def test_deregister_service_nonexistent(service_registry): +def test_unregister_service_nonexistent(service_registry): with pytest.raises(ValueError) as exc_info: - service_registry.deregister_service("nonexistent") + service_registry.unregister_service("nonexistent") assert str(exc_info.value) == "Service nonexistent not found." From 31cc39650d6ba8566fb48db6a10c736e4bc0f03c Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Fri, 13 Dec 2024 09:31:19 +0100 Subject: [PATCH 071/168] feat: implement lazy loading for ServiceRegistry classes in __init__.py --- .../service_registries/concrete/__init__.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkgs/swarmauri/swarmauri/service_registries/concrete/__init__.py b/pkgs/swarmauri/swarmauri/service_registries/concrete/__init__.py index e69de29bb..09ec2f608 100644 --- a/pkgs/swarmauri/swarmauri/service_registries/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/service_registries/concrete/__init__.py @@ -0,0 +1,16 @@ +from swarmauri.utils._lazy_import import _lazy_import + +# List of service_registry name (file names without the ".py" extension) and corresponding class names +service_registry_files = [ + ( + "swarmauri.service_registries.concrete.ServiceRegistry", + "ServiceRegistry", + ), +] + +# Lazy loading of service_registry classes, storing them in variables +for module_name, class_name in service_registry_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded service_registry classes to __all__ +__all__ = [class_name for _, class_name in service_registry_files] From e7f5a18122b226a70b022cab5e4e1b614bdfbe9e Mon Sep 17 00:00:00 2001 From: 3rd-Son Date: Fri, 13 Dec 2024 12:31:46 +0100 Subject: [PATCH 072/168] finisedh up pubsubtransport --- .../swarmauri/transport/base/TransportBase.py | 4 ++- .../transport/concrete/PubSubTransport.py | 36 +++++++++---------- .../transport/PubSubTransport_unit_test.py | 26 ++++++++++++-- 3 files changed, 43 insertions(+), 23 deletions(-) diff --git a/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py b/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py index 2020f677f..a25cff190 100644 --- a/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py +++ b/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py @@ -1,4 +1,5 @@ from typing import Dict, Any, List, Optional, Literal +from pydantic import ConfigDict, Field from enum import Enum, auto from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes from swarmauri_core.transport.ITransport import ITransport @@ -17,7 +18,8 @@ class TransportationProtocol(Enum): class TransportBase(ITransport, ComponentBase): allowed_protocols: List[TransportationProtocol] = [] - resource: Optional[str] = ResourceTypes.TRANSPORT.value + resource: Optional[str] = Field(default=ResourceTypes.TRANSPORT.value, frozen=True) + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) type: Literal["TransportBase"] = "TransportBase" def send(self, sender: str, recipient: str, message: Any) -> None: diff --git a/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py b/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py index ccc07c221..229114fb7 100644 --- a/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py +++ b/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py @@ -1,24 +1,14 @@ -from typing import Dict, Any, List, Optional, Set +from typing import Dict, Any, List, Optional, Set, Literal from uuid import uuid4 import asyncio -from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes -from swarmauri_core.transport.ITransport import ITransport from swarmauri.transport.base.TransportBase import TransportBase, TransportationProtocol -class PubSubTransport(TransportBase): +class PubSubTransport: allowed_protocols: List[TransportationProtocol] = [TransportationProtocol.PUBSUB] - - def __init__(self): - """ - Initialize the Publish-Subscribe Broker. - Manages topics and subscriptions for agents. - """ - super().__init__() - self._topics: Dict[str, Set[str]] = {} # Topic to subscriber mappings - self._subscribers: Dict[str, asyncio.Queue] = ( - {} - ) # Subscriber ID to message queue + _topics: Dict[str, Set[str]] = {} # Topic to subscriber mappings + _subscribers: Dict[str, asyncio.Queue] = {} + type: Literal["PubSubTransport"] = "PubSubTransport" async def subscribe(self, topic: str) -> str: """ @@ -30,9 +20,9 @@ async def subscribe(self, topic: str) -> str: Returns: str: Unique subscriber ID """ - subscriber_id = str(uuid4()) + subscriber_id = self.id - # Create message queue for this subscriber + # Create message queue for this subscribere self._subscribers[subscriber_id] = asyncio.Queue() # Add subscriber to topic @@ -42,7 +32,7 @@ async def subscribe(self, topic: str) -> str: return subscriber_id - async def unsubscribe(self, topic: str, subscriber_id: str): + async def unsubscribe(self, topic: str): """ Unsubscribe an agent from a topic. @@ -50,6 +40,7 @@ async def unsubscribe(self, topic: str, subscriber_id: str): topic (str): The topic to unsubscribe from subscriber_id (str): Unique identifier of the subscriber """ + subscriber_id = self.id if topic in self._topics and subscriber_id in self._topics[topic]: self._topics[topic].remove(subscriber_id) @@ -72,7 +63,7 @@ async def publish(self, topic: str, message: Any): for subscriber_id in self._topics[topic]: await self._subscribers[subscriber_id].put(message) - async def receive(self, subscriber_id: str) -> Any: + async def receive(self) -> Any: """ Receive messages for a specific subscriber. @@ -82,7 +73,7 @@ async def receive(self, subscriber_id: str) -> Any: Returns: Any: Received message """ - return await self._subscribers[subscriber_id].get() + return await self._subscribers[self.id].get() def send(self, sender: str, recipient: str, message: Any) -> None: """ @@ -120,3 +111,8 @@ def multicast(self, sender: str, recipients: List[str], message: Any) -> None: """ for topic in recipients: asyncio.create_task(self.publish(topic, message)) + + +check = PubSubTransport() +print(check.type) +print("I am okay") diff --git a/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py b/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py index 2d2c68f76..45ef9587c 100644 --- a/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py +++ b/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py @@ -6,12 +6,34 @@ PubSubTransport, ) from swarmauri.utils.timeout_wrapper import timeout +import logging @pytest.fixture -async def pubsub_transport(): +def pubsub_transport(): transport = PubSubTransport() - yield transport + return transport + + +@timeout(5) +@pytest.mark.unit +def test_ubc_resource(pubsub_transport): + assert pubsub_transport.resource == "Transport" + + +@timeout(5) +@pytest.mark.unit +def test_ubc_type(): + assert pubsub_transport.type == "PubSubTransport" + + +@timeout(5) +@pytest.mark.unit +def test_serialization(pubsub_transport): + assert ( + pubsub_transport.id + == PubSubTransport.model_validate_json(pubsub_transport.model_dump_json()).id + ) @timeout(5) From 857bdd0d5fc2bfb61c0a0433e9feeb1b0d49f6f2 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Fri, 13 Dec 2024 06:23:27 -0600 Subject: [PATCH 073/168] Update PubSubTransport.py --- pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py b/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py index 229114fb7..1bf1ea055 100644 --- a/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py +++ b/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py @@ -4,7 +4,7 @@ from swarmauri.transport.base.TransportBase import TransportBase, TransportationProtocol -class PubSubTransport: +class PubSubTransport(TransportBase): allowed_protocols: List[TransportationProtocol] = [TransportationProtocol.PUBSUB] _topics: Dict[str, Set[str]] = {} # Topic to subscriber mappings _subscribers: Dict[str, asyncio.Queue] = {} From 09ae2cb20f05790276c83146fbd387cec9319d9c Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Fri, 13 Dec 2024 06:24:16 -0600 Subject: [PATCH 074/168] Update PubSubTransport_unit_test.py --- .../swarmauri/tests/unit/transport/PubSubTransport_unit_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py b/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py index 45ef9587c..c8918b086 100644 --- a/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py +++ b/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py @@ -23,7 +23,7 @@ def test_ubc_resource(pubsub_transport): @timeout(5) @pytest.mark.unit -def test_ubc_type(): +def test_ubc_type(pubsub_transport): assert pubsub_transport.type == "PubSubTransport" From fc101dc0fb869f41e9e15bb7ed31072f30d3649a Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Fri, 13 Dec 2024 06:29:45 -0600 Subject: [PATCH 075/168] Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index 23ad0e37f..57341b401 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -15,7 +15,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.13" -swarmauri_core = "==0.5.3.dev3" +swarmauri_core = { path = "../core/swarmauri_core" } toml = "^0.10.2" httpx = "^0.27.0" joblib = "^1.4.0" From 172bc304725c06c43d7177d529711c60a167badf Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Fri, 13 Dec 2024 06:40:16 -0600 Subject: [PATCH 076/168] Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index 57341b401..2a806d207 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -15,7 +15,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.13" -swarmauri_core = { path = "../core/swarmauri_core" } +swarmauri_core = { path = "/pkgs/core/swarmauri_core" } toml = "^0.10.2" httpx = "^0.27.0" joblib = "^1.4.0" From a2b965fbeb7f283ad432d50dd9e505868572453a Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Fri, 13 Dec 2024 06:44:37 -0600 Subject: [PATCH 077/168] Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index 2a806d207..2270780ce 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -15,7 +15,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.13" -swarmauri_core = { path = "/pkgs/core/swarmauri_core" } +swarmauri_core = { path = "./pkgs/core/swarmauri_core" } toml = "^0.10.2" httpx = "^0.27.0" joblib = "^1.4.0" From e77b057ace2d49152078b133b4e63dbdbf0719a5 Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Fri, 13 Dec 2024 13:56:29 +0100 Subject: [PATCH 078/168] feat: add ControlPanel implementation and related interfaces --- pkgs/core/swarmauri_core/ComponentBase.py | 2 + .../control_panels/IControlPanel.py | 43 +++++++++ .../swarmauri_core/control_panels/__init__.py | 0 .../swarmauri/control_panels/__init__.py | 0 .../control_panels/base/ControlPanelBase.py | 62 ++++++++++++ .../swarmauri/control_panels/base/__init__.py | 0 .../control_panels/concrete/ControlPanel.py | 9 ++ .../control_panels/ControlPanel_unit_test.py | 96 +++++++++++++++++++ 8 files changed, 212 insertions(+) create mode 100644 pkgs/core/swarmauri_core/control_panels/IControlPanel.py create mode 100644 pkgs/core/swarmauri_core/control_panels/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/control_panels/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py create mode 100644 pkgs/swarmauri/swarmauri/control_panels/base/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/control_panels/concrete/ControlPanel.py create mode 100644 pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py diff --git a/pkgs/core/swarmauri_core/ComponentBase.py b/pkgs/core/swarmauri_core/ComponentBase.py index a5e558380..b5444fe72 100644 --- a/pkgs/core/swarmauri_core/ComponentBase.py +++ b/pkgs/core/swarmauri_core/ComponentBase.py @@ -58,6 +58,8 @@ class ResourceTypes(Enum): VCM = "VCM" DATA_CONNECTOR = "DataConnector" FACTORY = "Factory" + SERVICE_REGISTRY = "ServiceRegistry" + CONTROL_PANEL = "ControlPanel" def generate_id() -> str: diff --git a/pkgs/core/swarmauri_core/control_panels/IControlPanel.py b/pkgs/core/swarmauri_core/control_panels/IControlPanel.py new file mode 100644 index 000000000..e6fb53191 --- /dev/null +++ b/pkgs/core/swarmauri_core/control_panels/IControlPanel.py @@ -0,0 +1,43 @@ +from abc import ABC, abstractmethod +from typing import Any, List + + +class IControlPlane(ABC): + """ + Abstract base class for ControlPlane. + """ + + @abstractmethod + def create_agent(self, name: str, role: str) -> Any: + """ + Create an agent with the given name and role. + """ + pass + + @abstractmethod + def distribute_tasks(self, task: Any) -> None: + """ + Distribute tasks using the task strategy. + """ + pass + + @abstractmethod + def orchestrate_agents(self, task: Any) -> None: + """ + Orchestrate agents for task distribution. + """ + pass + + @abstractmethod + def remove_agent(self, name: str) -> None: + """ + Remove the agent with the specified name. + """ + pass + + @abstractmethod + def list_active_agents(self) -> List[str]: + """ + List all active agent names. + """ + pass diff --git a/pkgs/core/swarmauri_core/control_panels/__init__.py b/pkgs/core/swarmauri_core/control_panels/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/control_panels/__init__.py b/pkgs/swarmauri/swarmauri/control_panels/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py b/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py new file mode 100644 index 000000000..5202a8406 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py @@ -0,0 +1,62 @@ +from swarmauri_core.control_panels.IControlPanel import IControlPlane +from typing import Any, List, Literal +from pydantic import Field, ConfigDict +from swarmauri_core.ComponentBase import ResourceTypes +from swarmauri.service_registries.base.ServiceRegistryBase import ServiceRegistryBase +from swarmauri.factories.base.FactoryBase import FactoryBase +from swarmauri.task_strategies.base.TaskStrategyBase import TaskStrategyBase +from swarmauri.transports.base.TransportBase import TransportBase +from swarmauri_core.typing import SubclassUnion + + +class ControlPanelBase(IControlPlane): + """ + Implementation of the ControlPlane abstract class. + """ + + resource: ResourceTypes = Field(default=ResourceTypes.CONTROL_PANEL.value) + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + type: Literal["ControlPanelBase"] = "ControlPanelBase" + + agent_factory: SubclassUnion[FactoryBase] + service_registry: SubclassUnion[ServiceRegistryBase] + task_strategy: SubclassUnion[TaskStrategyBase] + transport: SubclassUnion[TransportBase] + + def create_agent(self, name: str, role: str) -> Any: + """ + Create an agent with the given name and role. + """ + agent = self.agent_factory.create_agent(name, role) + self.service_registry.register_service(name, {"role": role, "status": "active"}) + return agent + + def distribute_tasks(self, task: Any) -> None: + """ + Distribute tasks using the task strategy. + """ + self.task_strategy.assign_task(task, self.agent_factory, self.service_registry) + + def orchestrate_agents(self, task: Any) -> None: + """ + Orchestrate agents for task distribution. + """ + self.manage_agents() + self.distribute_tasks(task) + + def remove_agent(self, name: str) -> None: + """ + Remove the agent with the specified name. + """ + agent = self.agent_factory.get_agent_by_name(name) + if not agent: + raise ValueError(f"Agent {name} not found.") + self.service_registry.unregister_service(name) + self.agent_factory.delete_agent(name) + + def list_active_agents(self) -> List[str]: + """ + List all active agent names. + """ + agents = self.agent_factory.get_agents() + return [agent.name for agent in agents if agent] diff --git a/pkgs/swarmauri/swarmauri/control_panels/base/__init__.py b/pkgs/swarmauri/swarmauri/control_panels/base/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/control_panels/concrete/ControlPanel.py b/pkgs/swarmauri/swarmauri/control_panels/concrete/ControlPanel.py new file mode 100644 index 000000000..c8533ebc4 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/control_panels/concrete/ControlPanel.py @@ -0,0 +1,9 @@ +from typing import Literal +from swarmauri.control_panels.base.ControlPanelBase import ControlPanelBase + +class ControlPanel(ControlPanelBase): + """ + Concrete implementation of the ControlPanelBase. + """ + type: Literal["ControlPanel"] = "ControlPanel" + diff --git a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py new file mode 100644 index 000000000..83e180e6c --- /dev/null +++ b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py @@ -0,0 +1,96 @@ +import pytest +from unittest.mock import MagicMock, patch +from swarmauri.control_panels.concrete.ControlPanel import ControlPanel + + +@pytest.fixture +def control_panel(): + agent_factory = MagicMock() + service_registry = MagicMock() + task_strategy = MagicMock() + transport = MagicMock() + return ControlPanel(agent_factory, service_registry, task_strategy, transport) + + +def test_create_agent(control_panel): + # Arrange + control_panel.agent_factory.create_agent.return_value = "agent1" + name = "agent1" + role = "role1" + + # Act + result = control_panel.create_agent(name, role) + + # Assert + control_panel.agent_factory.create_agent.assert_called_with(name, role) + control_panel.service_registry.register_service.assert_called_with( + name, {"role": role, "status": "active"} + ) + assert result == "agent1" + + +def test_distribute_tasks(control_panel): + # Arrange + task = "task1" + + # Act + control_panel.distribute_tasks(task) + + # Assert + control_panel.task_strategy.assign_task.assert_called_with( + task, control_panel.agent_factory, control_panel.service_registry + ) + + +def test_orchestrate_agents(control_panel): + # Arrange + task = "task1" + + # Act + control_panel.orchestrate_agents(task) + + # Assert + with patch.object(control_panel, "manage_agents") as mock_manage_agents: + mock_manage_agents.assert_called_once() + control_panel.distribute_tasks.assert_called_with(task) + + +def test_remove_agent_success(control_panel): + # Arrange + name = "agent1" + agent = MagicMock() + control_panel.agent_factory.get_agent_by_name.return_value = agent + + # Act + control_panel.remove_agent(name) + + # Assert + control_panel.service_registry.unregister_service.assert_called_with(name) + control_panel.agent_factory.delete_agent.assert_called_with(name) + + +def test_remove_agent_not_found(control_panel): + # Arrange + name = "agent1" + control_panel.agent_factory.get_agent_by_name.return_value = None + + # Act & Assert + with pytest.raises(ValueError) as excinfo: + control_panel.remove_agent(name) + assert str(excinfo.value) == f"Agent {name} not found." + + +def test_list_active_agents(control_panel): + # Arrange + agent1 = MagicMock() + agent1.name = "agent1" + agent2 = MagicMock() + agent2.name = "agent2" + control_panel.agent_factory.get_agents.return_value = [agent1, agent2] + + # Act + result = control_panel.list_active_agents() + + # Assert + control_panel.agent_factory.get_agents.assert_called_once() + assert result == ["agent1", "agent2"] From a57dc3dd528609592b8c81f2ef18dbf7dc801d68 Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Mon, 16 Dec 2024 10:43:00 +0100 Subject: [PATCH 079/168] feat: implement task management strategies with base and round-robin strategy --- pkgs/core/swarmauri_core/ComponentBase.py | 1 + .../task_mgt_strategies/ITaskMgtStrategy.py | 43 +++++++++ .../task_mgt_strategies/__init__.py | 0 .../control_panels/base/ControlPanelBase.py | 7 +- .../control_panels/concrete/ControlPanel.py | 3 +- .../swarmauri/task_mgt_strategies/__init__.py | 0 .../base/TaskMgtStrategyBase.py | 63 +++++++++++++ .../task_mgt_strategies/base/__init__.py | 0 .../concrete/RoundRobinStrategy.py | 70 ++++++++++++++ .../task_mgt_strategies/concrete/__init__.py | 0 .../control_panels/ControlPanel_unit_test.py | 13 --- .../RoundRobinStrategy_unit_test.py | 93 +++++++++++++++++++ 12 files changed, 275 insertions(+), 18 deletions(-) create mode 100644 pkgs/core/swarmauri_core/task_mgt_strategies/ITaskMgtStrategy.py create mode 100644 pkgs/core/swarmauri_core/task_mgt_strategies/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/task_mgt_strategies/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/task_mgt_strategies/base/TaskMgtStrategyBase.py create mode 100644 pkgs/swarmauri/swarmauri/task_mgt_strategies/base/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/RoundRobinStrategy.py create mode 100644 pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/__init__.py create mode 100644 pkgs/swarmauri/tests/unit/task_mgt_strategies/RoundRobinStrategy_unit_test.py diff --git a/pkgs/core/swarmauri_core/ComponentBase.py b/pkgs/core/swarmauri_core/ComponentBase.py index f9be2bdcb..a72e62fac 100644 --- a/pkgs/core/swarmauri_core/ComponentBase.py +++ b/pkgs/core/swarmauri_core/ComponentBase.py @@ -60,6 +60,7 @@ class ResourceTypes(Enum): FACTORY = "Factory" SERVICE_REGISTRY = "ServiceRegistry" CONTROL_PANEL = "ControlPanel" + TASK_MGT_STRATEGY = "TaskMgtStrategy" def generate_id() -> str: diff --git a/pkgs/core/swarmauri_core/task_mgt_strategies/ITaskMgtStrategy.py b/pkgs/core/swarmauri_core/task_mgt_strategies/ITaskMgtStrategy.py new file mode 100644 index 000000000..13be06917 --- /dev/null +++ b/pkgs/core/swarmauri_core/task_mgt_strategies/ITaskMgtStrategy.py @@ -0,0 +1,43 @@ +from abc import ABC, abstractmethod +from typing import Any, Callable, Dict + + +class ITaskMgtStrategy(ABC): + """Abstract base class for TaskStrategy.""" + + @abstractmethod + def assign_task( + self, task: Dict[str, Any], agent_factory: Callable, service_registry: Callable + ) -> str: + """ + Abstract method to assign a task to a service. + """ + pass + + @abstractmethod + def add_task(self, task: Dict[str, Any]) -> None: + """ + Abstract method to add a task to the task queue. + """ + pass + + @abstractmethod + def remove_task(self, task_id: str) -> None: + """ + Abstract method to remove a task from the task queue. + """ + pass + + @abstractmethod + def get_task(self, task_id: str) -> Dict[str, Any]: + """ + Abstract method to get a task from the task queue. + """ + pass + + @abstractmethod + def process_tasks(self, task: Dict[str, Any]) -> None: + """ + Abstract method to process a task. + """ + pass diff --git a/pkgs/core/swarmauri_core/task_mgt_strategies/__init__.py b/pkgs/core/swarmauri_core/task_mgt_strategies/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py b/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py index 5202a8406..cee5b6c90 100644 --- a/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py +++ b/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py @@ -4,7 +4,7 @@ from swarmauri_core.ComponentBase import ResourceTypes from swarmauri.service_registries.base.ServiceRegistryBase import ServiceRegistryBase from swarmauri.factories.base.FactoryBase import FactoryBase -from swarmauri.task_strategies.base.TaskStrategyBase import TaskStrategyBase +from swarmauri.task_mgt_strategies.base.TaskMgtStrategyBase import TaskMgtStrategyBase from swarmauri.transports.base.TransportBase import TransportBase from swarmauri_core.typing import SubclassUnion @@ -20,7 +20,7 @@ class ControlPanelBase(IControlPlane): agent_factory: SubclassUnion[FactoryBase] service_registry: SubclassUnion[ServiceRegistryBase] - task_strategy: SubclassUnion[TaskStrategyBase] + task_mgt_strategy: SubclassUnion[TaskMgtStrategyBase] transport: SubclassUnion[TransportBase] def create_agent(self, name: str, role: str) -> Any: @@ -35,13 +35,12 @@ def distribute_tasks(self, task: Any) -> None: """ Distribute tasks using the task strategy. """ - self.task_strategy.assign_task(task, self.agent_factory, self.service_registry) + self.task_mgt_strategy.assign_task(task, self.service_registry) def orchestrate_agents(self, task: Any) -> None: """ Orchestrate agents for task distribution. """ - self.manage_agents() self.distribute_tasks(task) def remove_agent(self, name: str) -> None: diff --git a/pkgs/swarmauri/swarmauri/control_panels/concrete/ControlPanel.py b/pkgs/swarmauri/swarmauri/control_panels/concrete/ControlPanel.py index c8533ebc4..14a7b2205 100644 --- a/pkgs/swarmauri/swarmauri/control_panels/concrete/ControlPanel.py +++ b/pkgs/swarmauri/swarmauri/control_panels/concrete/ControlPanel.py @@ -1,9 +1,10 @@ from typing import Literal from swarmauri.control_panels.base.ControlPanelBase import ControlPanelBase + class ControlPanel(ControlPanelBase): """ Concrete implementation of the ControlPanelBase. """ + type: Literal["ControlPanel"] = "ControlPanel" - diff --git a/pkgs/swarmauri/swarmauri/task_mgt_strategies/__init__.py b/pkgs/swarmauri/swarmauri/task_mgt_strategies/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/task_mgt_strategies/base/TaskMgtStrategyBase.py b/pkgs/swarmauri/swarmauri/task_mgt_strategies/base/TaskMgtStrategyBase.py new file mode 100644 index 000000000..698751f71 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/task_mgt_strategies/base/TaskMgtStrategyBase.py @@ -0,0 +1,63 @@ +from abc import abstractmethod + +from pydantic import ConfigDict, Field +from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes +from swarmauri_core.task_mgt_strategies.ITaskMgtStrategy import ITaskMgtStrategy +from typing import Any, Callable, Dict, Literal, Optional + + +class TaskMgtStrategyBase(ITaskMgtStrategy, ComponentBase): + """Base class for TaskStrategy.""" + + type: Literal["TaskMgtStrategyBase"] = "TaskMgtStrategyBase" + resource: Optional[str] = Field( + default=ResourceTypes.TASK_MGT_STRATEGY.value, frozen=True + ) + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + + @abstractmethod + def assign_task( + self, task: Dict[str, Any], agent_factory: Callable, service_registry: Callable + ) -> str: + """ + Abstract method to assign a task to a service. + """ + raise NotImplementedError( + "assign_task method must be implemented in derived classes." + ) + + @abstractmethod + def add_task(self, task: Dict[str, Any]) -> None: + """ + Abstract method to add a task to the task queue. + """ + raise NotImplementedError( + "add_task method must be implemented in derived classes." + ) + + @abstractmethod + def remove_task(self, task_id: str) -> None: + """ + Abstract method to remove a task from the task queue. + """ + raise NotImplementedError( + "remove_task method must be implemented in derived classes." + ) + + @abstractmethod + def get_task(self, task_id: str) -> Dict[str, Any]: + """ + Abstract method to get a task from the task queue. + """ + raise NotImplementedError( + "get_task method must be implemented in derived classes." + ) + + @abstractmethod + def process_tasks(self, task: Dict[str, Any]) -> None: + """ + Abstract method to process tasks. + """ + raise NotImplementedError( + "process_task method must be implemented in derived classes." + ) diff --git a/pkgs/swarmauri/swarmauri/task_mgt_strategies/base/__init__.py b/pkgs/swarmauri/swarmauri/task_mgt_strategies/base/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/RoundRobinStrategy.py b/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/RoundRobinStrategy.py new file mode 100644 index 000000000..53b1bddd6 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/RoundRobinStrategy.py @@ -0,0 +1,70 @@ +from typing import Callable, Dict, Any, List +from swarmauri.task_mgt_strategies.base.TaskMgtStrategyBase import TaskMgtStrategyBase +from queue import Queue +import logging + + +class RoundRobinStrategy(TaskMgtStrategyBase): + """Round-robin task assignment strategy.""" + + task_queue: Queue = Queue() # Synchronous task queue for incoming tasks + task_assignments: Dict[str, str] = {} # Tracks task assignments + current_index: int = 0 # Tracks the next service to assign tasks to + + def assign_task(self, task: Dict[str, Any], service_registry: Callable[[], List[str]]) -> None: + """ + Assign a task to a service using the round-robin strategy. + :param task: Task metadata and payload. + :param service_registry: Callable that returns available services. + """ + available_services = service_registry() + if not available_services: + raise ValueError("No services available for task assignment.") + + # Select the service based on the round-robin index + service = available_services[self.current_index % len(available_services)] + self.task_assignments[task["task_id"]] = service + self.current_index += 1 + logging.info(f"Task '{task['task_id']}' assigned to service '{service}'.") + + def add_task(self, task: Dict[str, Any]) -> None: + """ + Add a task to the task queue. + :param task: Task metadata and payload. + """ + self.task_queue.put(task) + + def remove_task(self, task_id: str) -> None: + """ + Remove a task from the task registry. + :param task_id: Unique identifier of the task to remove. + """ + if task_id in self.task_assignments: + del self.task_assignments[task_id] + logging.info(f"Task '{task_id}' removed from assignments.") + else: + raise ValueError(f"Task '{task_id}' not found in assignments.") + + def get_task(self, task_id: str) -> Dict[str, Any]: + """ + Get a task's assigned service. + :param task_id: Unique identifier of the task. + :return: Task assignment details. + """ + if task_id in self.task_assignments: + service = self.task_assignments[task_id] + return {"task_id": task_id, "assigned_service": service} + else: + raise ValueError(f"Task '{task_id}' not found in assignments.") + + def process_tasks(self, service_registry: Callable[[], List[str]]) -> None: + """ + Process tasks from the task queue and assign them to services. + :param service_registry: Callable that returns available services. + """ + while not self.task_queue.empty(): + task = self.task_queue.get() + try: + self.assign_task(task, service_registry) + except ValueError as e: + raise ValueError(f"Error assigning task: {e}") diff --git a/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/__init__.py b/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py index 83e180e6c..327f97e34 100644 --- a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py +++ b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py @@ -13,15 +13,12 @@ def control_panel(): def test_create_agent(control_panel): - # Arrange control_panel.agent_factory.create_agent.return_value = "agent1" name = "agent1" role = "role1" - # Act result = control_panel.create_agent(name, role) - # Assert control_panel.agent_factory.create_agent.assert_called_with(name, role) control_panel.service_registry.register_service.assert_called_with( name, {"role": role, "status": "active"} @@ -30,26 +27,20 @@ def test_create_agent(control_panel): def test_distribute_tasks(control_panel): - # Arrange task = "task1" - # Act control_panel.distribute_tasks(task) - # Assert control_panel.task_strategy.assign_task.assert_called_with( task, control_panel.agent_factory, control_panel.service_registry ) def test_orchestrate_agents(control_panel): - # Arrange task = "task1" - # Act control_panel.orchestrate_agents(task) - # Assert with patch.object(control_panel, "manage_agents") as mock_manage_agents: mock_manage_agents.assert_called_once() control_panel.distribute_tasks.assert_called_with(task) @@ -61,10 +52,8 @@ def test_remove_agent_success(control_panel): agent = MagicMock() control_panel.agent_factory.get_agent_by_name.return_value = agent - # Act control_panel.remove_agent(name) - # Assert control_panel.service_registry.unregister_service.assert_called_with(name) control_panel.agent_factory.delete_agent.assert_called_with(name) @@ -88,9 +77,7 @@ def test_list_active_agents(control_panel): agent2.name = "agent2" control_panel.agent_factory.get_agents.return_value = [agent1, agent2] - # Act result = control_panel.list_active_agents() - # Assert control_panel.agent_factory.get_agents.assert_called_once() assert result == ["agent1", "agent2"] diff --git a/pkgs/swarmauri/tests/unit/task_mgt_strategies/RoundRobinStrategy_unit_test.py b/pkgs/swarmauri/tests/unit/task_mgt_strategies/RoundRobinStrategy_unit_test.py new file mode 100644 index 000000000..d28d9ab3c --- /dev/null +++ b/pkgs/swarmauri/tests/unit/task_mgt_strategies/RoundRobinStrategy_unit_test.py @@ -0,0 +1,93 @@ +import pytest +from swarmauri.task_mgt_strategies.concrete.RoundRobinStrategy import RoundRobinStrategy +from unittest.mock import MagicMock + + +@pytest.fixture +def round_robin_strategy(): + """Fixture to create a RoundRobinStrategy instance.""" + return RoundRobinStrategy() + + +def test_assign_task(round_robin_strategy): + # Setup + task = {"task_id": "task1", "payload": "data"} + service_registry = MagicMock(return_value=["service1", "service2"]) + + round_robin_strategy.assign_task(task, service_registry) + + assert round_robin_strategy.task_assignments["task1"] == "service1" + assert round_robin_strategy.current_index == 1 + + +def test_assign_task_no_services(round_robin_strategy): + # Setup + task = {"task_id": "task1", "payload": "data"} + service_registry = MagicMock(return_value=[]) + + # Execute & Verify + with pytest.raises(ValueError) as exc_info: + round_robin_strategy.assign_task(task, service_registry) + assert str(exc_info.value) == "No services available for task assignment." + + +def test_add_task(round_robin_strategy): + task = {"task_id": "task1", "payload": "data"} + + round_robin_strategy.add_task(task) + + assert not round_robin_strategy.task_queue.empty() + queued_task = round_robin_strategy.task_queue.get() + assert queued_task == task + + +def test_remove_task(round_robin_strategy): + task_id = "task1" + round_robin_strategy.task_assignments[task_id] = "service1" + + round_robin_strategy.remove_task(task_id) + + assert task_id not in round_robin_strategy.task_assignments + + +def test_remove_task_not_found(round_robin_strategy): + task_id = "task1" + + with pytest.raises(ValueError) as exc_info: + round_robin_strategy.remove_task(task_id) + assert str(exc_info.value) == "Task 'task1' not found in assignments." + + +def test_get_task(round_robin_strategy): + task_id = "task1" + round_robin_strategy.task_assignments[task_id] = "service1" + + result = round_robin_strategy.get_task(task_id) + + assert result == {"task_id": task_id, "assigned_service": "service1"} + + +def test_get_task_not_found(round_robin_strategy): + task_id = "task1" + + with pytest.raises(ValueError) as exc_info: + round_robin_strategy.get_task(task_id) + assert str(exc_info.value) == "Task 'task1' not found in assignments." + + +def test_process_tasks(round_robin_strategy): + service_registry = MagicMock(return_value=["service1", "service2"]) + tasks = [ + {"task_id": "task1", "payload": "data1"}, + {"task_id": "task2", "payload": "data2"}, + {"task_id": "task3", "payload": "data3"}, + ] + for task in tasks: + round_robin_strategy.add_task(task) + + round_robin_strategy.process_tasks(service_registry) + + assert round_robin_strategy.task_assignments["task1"] == "service1" + assert round_robin_strategy.task_assignments["task2"] == "service2" + assert round_robin_strategy.task_assignments["task3"] == "service1" + assert round_robin_strategy.current_index == 3 From de249fc5a8f3fd8310b9e4c26d559382168d6034 Mon Sep 17 00:00:00 2001 From: 3rd-Son Date: Mon, 16 Dec 2024 12:27:25 +0100 Subject: [PATCH 080/168] implemented pipelines --- pkgs/core/swarmauri_core/ComponentBase.py | 1 + .../swarmauri_core/pipelines/IPipeline.py | 57 ++++++++++ .../core/swarmauri_core/pipelines/__init__.py | 0 .../swarmauri/pipelines/base/PipelineBase.py | 105 ++++++++++++++++++ .../swarmauri/pipelines/base/__init__.py | 0 .../swarmauri/pipelines/concrete/Pipeline.py | 50 +++++++++ .../swarmauri/pipelines/concrete/__init__.py | 13 +++ .../unit/pipelines/Pipeline_unit_test.py | 84 ++++++++++++++ 8 files changed, 310 insertions(+) create mode 100644 pkgs/core/swarmauri_core/pipelines/IPipeline.py create mode 100644 pkgs/core/swarmauri_core/pipelines/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/pipelines/base/PipelineBase.py create mode 100644 pkgs/swarmauri/swarmauri/pipelines/base/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/pipelines/concrete/Pipeline.py create mode 100644 pkgs/swarmauri/swarmauri/pipelines/concrete/__init__.py create mode 100644 pkgs/swarmauri/tests/unit/pipelines/Pipeline_unit_test.py diff --git a/pkgs/core/swarmauri_core/ComponentBase.py b/pkgs/core/swarmauri_core/ComponentBase.py index ed142c918..2513f0d88 100644 --- a/pkgs/core/swarmauri_core/ComponentBase.py +++ b/pkgs/core/swarmauri_core/ComponentBase.py @@ -58,6 +58,7 @@ class ResourceTypes(Enum): VCM = "VCM" DATA_CONNECTOR = "DataConnector" FACTORY = "Factory" + PIPELINE = "Pipeline" def generate_id() -> str: diff --git a/pkgs/core/swarmauri_core/pipelines/IPipeline.py b/pkgs/core/swarmauri_core/pipelines/IPipeline.py new file mode 100644 index 000000000..9e941fdd9 --- /dev/null +++ b/pkgs/core/swarmauri_core/pipelines/IPipeline.py @@ -0,0 +1,57 @@ +from abc import ABC, abstractmethod +from typing import Any, Callable, List +from enum import Enum + + +class PipelineStatus(Enum): + """ + Enum representing the status of a pipeline execution. + """ + + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + STOPPED = "stopped" + + +class IPipeline(ABC): + """ + Interface defining core methods for pipeline execution and management. + """ + + @abstractmethod + def add_task(self, task: Callable, *args: Any, **kwargs: Any) -> None: + """ + Add a task to the pipeline. + + :param task: Callable task to be executed + :param args: Positional arguments for the task + :param kwargs: Keyword arguments for the task + """ + pass + + @abstractmethod + def execute(self, *args: Any, **kwargs: Any) -> List[Any]: + """ + Execute the entire pipeline. + + :return: List of results from pipeline execution + """ + pass + + @abstractmethod + def get_status(self) -> PipelineStatus: + """ + Get the current status of the pipeline. + + :return: Current pipeline status + """ + pass + + @abstractmethod + def reset(self) -> None: + """ + Reset the pipeline to its initial state. + """ + pass diff --git a/pkgs/core/swarmauri_core/pipelines/__init__.py b/pkgs/core/swarmauri_core/pipelines/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/pipelines/base/PipelineBase.py b/pkgs/swarmauri/swarmauri/pipelines/base/PipelineBase.py new file mode 100644 index 000000000..da91427b9 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/pipelines/base/PipelineBase.py @@ -0,0 +1,105 @@ +from typing import Any, Callable, List, Optional, Dict +from pydantic import BaseModel, ConfigDict, Field +from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes +from swarmauri_core.pipelines.IPipeline import IPipeline, PipelineStatus +import uuid + + +class PipelineBase(ComponentBase, IPipeline): + """ + Base class providing default behavior for task orchestration, + error handling, and result aggregation. + """ + + resource: Optional[str] = Field(default=ResourceTypes.PIPELINE.value, frozen=True) + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + type: str = "PipelineBase" + + # Pydantic model fields + id: str = Field(default_factory=lambda: str(uuid.uuid4())) + tasks: List[Dict[str, Any]] = Field(default_factory=list) + parallel: bool = Field(default=False) + + def __init__( + self, tasks: Optional[List[Dict[str, Any]]] = None, parallel: bool = False + ): + """ + Initialize the pipeline. + + :param tasks: Optional list of tasks to initialize pipeline with + :param parallel: Flag to indicate parallel or sequential execution + """ + super().__init__() + self.tasks = tasks or [] + self._results: List[Any] = [] + self._status: PipelineStatus = PipelineStatus.PENDING + self.parallel = parallel + + def add_task(self, task: Callable, *args: Any, **kwargs: Any) -> None: + """ + Add a task to the pipeline. + + :param task: Callable task to be executed + :param args: Positional arguments for the task + :param kwargs: Keyword arguments for the task + """ + task_entry = {"callable": task, "args": args, "kwargs": kwargs} + self.tasks.append(task_entry) + + def execute(self, *args: Any, **kwargs: Any) -> List[Any]: + """ + Execute pipeline tasks. + + :return: List of results from pipeline execution + """ + try: + self._status = PipelineStatus.RUNNING + self._results = [] + + if self.parallel: + # Implement parallel execution logic + from concurrent.futures import ThreadPoolExecutor + + with ThreadPoolExecutor() as executor: + futures = [ + executor.submit( + task["callable"], *task["args"], **task["kwargs"] + ) + for task in self.tasks + ] + self._results = [future.result() for future in futures] + else: + # Sequential execution + for task in self.tasks: + result = task["callable"](*task["args"], **task["kwargs"]) + self._results.append(result) + + self._status = PipelineStatus.COMPLETED + return self._results + + except Exception as e: + self._status = PipelineStatus.FAILED + raise RuntimeError(f"Pipeline execution failed: {e}") + + def get_status(self) -> PipelineStatus: + """ + Get the current status of the pipeline. + + :return: Current pipeline status + """ + return self._status + + def reset(self) -> None: + """ + Reset the pipeline to its initial state. + """ + self._results = [] + self._status = PipelineStatus.PENDING + + def get_results(self) -> List[Any]: + """ + Get the results of the pipeline execution. + + :return: List of results + """ + return self._results diff --git a/pkgs/swarmauri/swarmauri/pipelines/base/__init__.py b/pkgs/swarmauri/swarmauri/pipelines/base/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/pipelines/concrete/Pipeline.py b/pkgs/swarmauri/swarmauri/pipelines/concrete/Pipeline.py new file mode 100644 index 000000000..82c4ec593 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/pipelines/concrete/Pipeline.py @@ -0,0 +1,50 @@ +from typing import Any, Callable, List, Optional, Dict +from swarmauri.pipelines.base.PipelineBase import PipelineBase + + +class Pipeline(PipelineBase): + """ + Concrete implementation of a pipeline with additional + customization options. + """ + + type: str = "Pipeline" + + def __init__( + self, + tasks: Optional[List[Dict[str, Any]]] = None, + parallel: bool = False, + error_handler: Optional[Callable[[Exception], Any]] = None, + ): + """ + Initialize a customizable pipeline. + + :param tasks: Optional list of tasks to initialize pipeline with + :param parallel: Flag to indicate parallel or sequential execution + :param error_handler: Optional custom error handling function + """ + super().__init__(tasks, parallel) + self._error_handler = error_handler + + def execute(self, *args: Any, **kwargs: Any) -> List[Any]: + """ + Execute pipeline with optional custom error handling. + + :return: List of results from pipeline execution + """ + try: + return super().execute(*args, **kwargs) + except Exception as e: + if self._error_handler: + return [self._error_handler(e)] + raise + + def with_error_handler(self, handler: Callable[[Exception], Any]) -> "Pipeline": + """ + Add a custom error handler to the pipeline. + + :param handler: Error handling function + :return: Current pipeline instance + """ + self._error_handler = handler + return self diff --git a/pkgs/swarmauri/swarmauri/pipelines/concrete/__init__.py b/pkgs/swarmauri/swarmauri/pipelines/concrete/__init__.py new file mode 100644 index 000000000..db1142f92 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/pipelines/concrete/__init__.py @@ -0,0 +1,13 @@ +from swarmauri.utils._lazy_import import _lazy_import + +# List of pipeline names (file names without the ".py" extension) and corresponding class names +pipeline_files = [ + ("swarmauri.pipelines.concrete.Pipeline", "Pipeline"), +] + +# Lazy loading of pipeline classes, storing them in variables +for module_name, class_name in pipeline_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded pipeline classes to __all__ +__all__ = [class_name for _, class_name in pipeline_files] diff --git a/pkgs/swarmauri/tests/unit/pipelines/Pipeline_unit_test.py b/pkgs/swarmauri/tests/unit/pipelines/Pipeline_unit_test.py new file mode 100644 index 000000000..359d9d759 --- /dev/null +++ b/pkgs/swarmauri/tests/unit/pipelines/Pipeline_unit_test.py @@ -0,0 +1,84 @@ +import pytest +from swarmauri.pipelines.concrete.Pipeline import Pipeline +from swarmauri_core.pipelines.IPipeline import PipelineStatus + + +@pytest.fixture(scope="module") +def simple_tasks(): + def task1(): + return "Task 1 completed" + + def task2(x): + return f"Task 2 with {x}" + + def task3(x, y): + return x + y + + return [task1, task2, task3] + + +@pytest.fixture(scope="module") +def pipeline(simple_tasks): + pipeline = Pipeline() + pipeline.add_task(simple_tasks[0]) + pipeline.add_task(simple_tasks[1], "parameter") + pipeline.add_task(simple_tasks[2], 10, 20) + return pipeline + + +@pytest.mark.unit +def test_ubc_resource(pipeline): + assert pipeline.resource == "Pipeline" + + +@pytest.mark.unit +def test_ubc_type(pipeline): + assert pipeline.type == "Pipeline" + + +@pytest.mark.unit +def test_serialization(pipeline): + assert pipeline.id == Pipeline.model_validate_json(pipeline.model_dump_json()).id + + +@pytest.mark.unit +def test_pipeline_initial_status(pipeline): + assert pipeline.get_status() == PipelineStatus.PENDING + + +@pytest.mark.unit +def test_pipeline_execution(pipeline): + results = pipeline.execute() + + assert len(results) == 3 + assert results[0] == "Task 1 completed" + assert results[1] == "Task 2 with parameter" + assert results[2] == 30 + assert pipeline.get_status() == PipelineStatus.COMPLETED + + +@pytest.mark.unit +def test_pipeline_reset(pipeline): + pipeline.reset() + assert pipeline.get_status() == PipelineStatus.PENDING + assert len(pipeline.get_results()) == 0 + + +@pytest.mark.unit +def test_pipeline_add_task(simple_tasks): + pipeline = Pipeline() + initial_task_count = len(pipeline.tasks) + + pipeline.add_task(simple_tasks[0]) + assert len(pipeline.tasks) == initial_task_count + 1 + + +@pytest.mark.unit +def test_pipeline_get_results(simple_tasks): + pipeline = Pipeline() + pipeline.add_task(simple_tasks[0]) + pipeline.execute() + + results = pipeline.get_results() + assert len(results) == 1 + assert results[0] == "Task 1 completed" From 3f0c3dc439b93591a494a20c75b765a191abcf46 Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Mon, 16 Dec 2024 13:21:55 +0100 Subject: [PATCH 081/168] feat: enhance RoundRobinStrategy and ControlPanelBase with task processing and agent management improvements --- .../control_panels/IControlPanel.py | 30 ++- .../control_panels/base/ControlPanelBase.py | 70 +++++-- .../concrete/RoundRobinStrategy.py | 5 +- .../control_panels/ControlPanel_unit_test.py | 179 +++++++++++++----- .../RoundRobinStrategy_unit_test.py | 45 ++++- 5 files changed, 253 insertions(+), 76 deletions(-) diff --git a/pkgs/core/swarmauri_core/control_panels/IControlPanel.py b/pkgs/core/swarmauri_core/control_panels/IControlPanel.py index e6fb53191..d1810b023 100644 --- a/pkgs/core/swarmauri_core/control_panels/IControlPanel.py +++ b/pkgs/core/swarmauri_core/control_panels/IControlPanel.py @@ -15,29 +15,43 @@ def create_agent(self, name: str, role: str) -> Any: pass @abstractmethod - def distribute_tasks(self, task: Any) -> None: + def remove_agent(self, name: str) -> None: """ - Distribute tasks using the task strategy. + Remove the agent with the specified name. """ pass @abstractmethod - def orchestrate_agents(self, task: Any) -> None: + def list_active_agents(self) -> List[str]: """ - Orchestrate agents for task distribution. + List all active agent names. """ pass @abstractmethod - def remove_agent(self, name: str) -> None: + def submit_tasks(self, tasks: List[Any]) -> None: """ - Remove the agent with the specified name. + Submit one or more tasks to the task management strategy for processing. """ pass @abstractmethod - def list_active_agents(self) -> List[str]: + def process_tasks(self) -> None: """ - List all active agent names. + Process and assign tasks from the queue, then transport them to their assigned services. + """ + pass + + @abstractmethod + def distribute_tasks(self, task: Any) -> None: + """ + Distribute tasks using the task strategy. + """ + pass + + @abstractmethod + def orchestrate_agents(self, task: Any) -> None: + """ + Orchestrate agents for task distribution. """ pass diff --git a/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py b/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py index cee5b6c90..41653005c 100644 --- a/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py +++ b/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py @@ -7,11 +7,14 @@ from swarmauri.task_mgt_strategies.base.TaskMgtStrategyBase import TaskMgtStrategyBase from swarmauri.transports.base.TransportBase import TransportBase from swarmauri_core.typing import SubclassUnion +import logging class ControlPanelBase(IControlPlane): """ Implementation of the ControlPlane abstract class. + This class orchestrates agents, manages tasks, and ensures task distribution + and transport between agents and services. """ resource: ResourceTypes = Field(default=ResourceTypes.CONTROL_PANEL.value) @@ -23,39 +26,74 @@ class ControlPanelBase(IControlPlane): task_mgt_strategy: SubclassUnion[TaskMgtStrategyBase] transport: SubclassUnion[TransportBase] + # Agent management methods def create_agent(self, name: str, role: str) -> Any: """ - Create an agent with the given name and role. + Create an agent with the given name and role, and register it in the service registry. """ agent = self.agent_factory.create_agent(name, role) self.service_registry.register_service(name, {"role": role, "status": "active"}) + logging.info(f"Agent '{name}' with role '{role}' created and registered.") return agent - def distribute_tasks(self, task: Any) -> None: - """ - Distribute tasks using the task strategy. - """ - self.task_mgt_strategy.assign_task(task, self.service_registry) - - def orchestrate_agents(self, task: Any) -> None: - """ - Orchestrate agents for task distribution. - """ - self.distribute_tasks(task) - def remove_agent(self, name: str) -> None: """ - Remove the agent with the specified name. + Remove the agent with the specified name and unregister it from the service registry. """ agent = self.agent_factory.get_agent_by_name(name) if not agent: - raise ValueError(f"Agent {name} not found.") + raise ValueError(f"Agent '{name}' not found.") self.service_registry.unregister_service(name) self.agent_factory.delete_agent(name) + logging.info(f"Agent '{name}' removed and unregistered.") def list_active_agents(self) -> List[str]: """ List all active agent names. """ agents = self.agent_factory.get_agents() - return [agent.name for agent in agents if agent] + active_agents = [agent.name for agent in agents if agent] + logging.info(f"Active agents listed: {active_agents}") + return active_agents + + # Task management methods + def submit_tasks(self, tasks: List[Any]) -> None: + """ + Submit one or more tasks to the task management strategy for processing. + """ + for task in tasks: + self.task_mgt_strategy.add_task(task) + logging.info( + f"Task '{task.get('task_id', 'unknown')}' submitted to the strategy." + ) + + def process_tasks(self) -> None: + """ + Process and assign tasks from the queue, then transport them to their assigned services. + """ + try: + self.task_mgt_strategy.process_tasks( + self.service_registry.get_services, self.transport + ) + logging.info("Tasks processed and transported successfully.") + except Exception as e: + logging.error(f"Error while processing tasks: {e}") + raise ValueError(f"Error processing tasks: {e}") + + def distribute_tasks(self, task: Any) -> None: + """ + Distribute tasks using the task strategy (manual or on-demand assignment). + """ + self.task_mgt_strategy.assign_task(task, self.service_registry.get_services) + logging.info( + f"Task '{task.get('task_id', 'unknown')}' distributed to a service." + ) + + # Orchestration method + def orchestrate_agents(self, tasks: List[Any]) -> None: + """ + Orchestrate agents for task distribution and transportation. + """ + self.submit_tasks(tasks) # Add task to the strategy + self.process_tasks() # Process and transport the task + logging.info("Agents orchestrated successfully.") diff --git a/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/RoundRobinStrategy.py b/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/RoundRobinStrategy.py index 53b1bddd6..d04c5a442 100644 --- a/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/RoundRobinStrategy.py +++ b/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/RoundRobinStrategy.py @@ -57,14 +57,17 @@ def get_task(self, task_id: str) -> Dict[str, Any]: else: raise ValueError(f"Task '{task_id}' not found in assignments.") - def process_tasks(self, service_registry: Callable[[], List[str]]) -> None: + def process_tasks(self, service_registry: Callable[[], List[str]], transport: Callable) -> None: """ Process tasks from the task queue and assign them to services. :param service_registry: Callable that returns available services. + :param transport: Callable used to send tasks to assigned services. """ while not self.task_queue.empty(): task = self.task_queue.get() try: self.assign_task(task, service_registry) + assigned_service = self.task_assignments[task["task_id"]] + transport.send(task, assigned_service) except ValueError as e: raise ValueError(f"Error assigning task: {e}") diff --git a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py index 327f97e34..8ce020ae0 100644 --- a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py +++ b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py @@ -1,83 +1,166 @@ import pytest -from unittest.mock import MagicMock, patch -from swarmauri.control_panels.concrete.ControlPanel import ControlPanel +from unittest.mock import MagicMock +from swarmauri.control_panels.base.ControlPanelBase import ControlPanelBase +from swarmauri.factories.base.FactoryBase import FactoryBase +from swarmauri.service_registries.base.ServiceRegistryBase import ServiceRegistryBase +from swarmauri.task_mgt_strategies.base.TaskMgtStrategyBase import TaskMgtStrategyBase +from swarmauri.transports.base.TransportBase import TransportBase @pytest.fixture def control_panel(): - agent_factory = MagicMock() - service_registry = MagicMock() - task_strategy = MagicMock() - transport = MagicMock() - return ControlPanel(agent_factory, service_registry, task_strategy, transport) + """Fixture to create a ControlPanelBase instance with mocked dependencies.""" + agent_factory = MagicMock(spec=FactoryBase) + service_registry = MagicMock(spec=ServiceRegistryBase) + task_mgt_strategy = MagicMock(spec=TaskMgtStrategyBase) + transport = MagicMock(spec=TransportBase) + + return ControlPanelBase( + agent_factory=agent_factory, + service_registry=service_registry, + task_mgt_strategy=task_mgt_strategy, + transport=transport, + ) def test_create_agent(control_panel): - control_panel.agent_factory.create_agent.return_value = "agent1" - name = "agent1" - role = "role1" - - result = control_panel.create_agent(name, role) - - control_panel.agent_factory.create_agent.assert_called_with(name, role) - control_panel.service_registry.register_service.assert_called_with( - name, {"role": role, "status": "active"} - ) - assert result == "agent1" + """Test the create_agent method.""" + agent_name = "agent1" + agent_role = "worker" + agent = MagicMock() + # Configure mocks + control_panel.agent_factory.create_agent.return_value = agent -def test_distribute_tasks(control_panel): - task = "task1" + # Call the method + result = control_panel.create_agent(agent_name, agent_role) - control_panel.distribute_tasks(task) - - control_panel.task_strategy.assign_task.assert_called_with( - task, control_panel.agent_factory, control_panel.service_registry + # Assertions + control_panel.agent_factory.create_agent.assert_called_once_with( + agent_name, agent_role ) + control_panel.service_registry.register_service.assert_called_once_with( + agent_name, {"role": agent_role, "status": "active"} + ) + assert result == agent -def test_orchestrate_agents(control_panel): - task = "task1" - - control_panel.orchestrate_agents(task) - - with patch.object(control_panel, "manage_agents") as mock_manage_agents: - mock_manage_agents.assert_called_once() - control_panel.distribute_tasks.assert_called_with(task) - - -def test_remove_agent_success(control_panel): - # Arrange - name = "agent1" +def test_remove_agent(control_panel): + """Test the remove_agent method.""" + agent_name = "agent1" agent = MagicMock() + + # Configure mocks control_panel.agent_factory.get_agent_by_name.return_value = agent - control_panel.remove_agent(name) + # Call the method + control_panel.remove_agent(agent_name) - control_panel.service_registry.unregister_service.assert_called_with(name) - control_panel.agent_factory.delete_agent.assert_called_with(name) + # Assertions + control_panel.agent_factory.get_agent_by_name.assert_called_once_with(agent_name) + control_panel.service_registry.unregister_service.assert_called_once_with( + agent_name + ) + control_panel.agent_factory.delete_agent.assert_called_once_with(agent_name) def test_remove_agent_not_found(control_panel): - # Arrange - name = "agent1" + """Test remove_agent when the agent is not found.""" + agent_name = "agent1" + + # Configure mocks control_panel.agent_factory.get_agent_by_name.return_value = None - # Act & Assert - with pytest.raises(ValueError) as excinfo: - control_panel.remove_agent(name) - assert str(excinfo.value) == f"Agent {name} not found." + # Call the method and expect a ValueError + with pytest.raises(ValueError) as exc_info: + control_panel.remove_agent(agent_name) + assert str(exc_info.value) == f"Agent '{agent_name}' not found." def test_list_active_agents(control_panel): - # Arrange + """Test the list_active_agents method.""" agent1 = MagicMock() agent1.name = "agent1" agent2 = MagicMock() agent2.name = "agent2" - control_panel.agent_factory.get_agents.return_value = [agent1, agent2] + agents = [agent1, agent2] + # Configure mocks + control_panel.agent_factory.get_agents.return_value = agents + + # Call the method result = control_panel.list_active_agents() + # Assertions control_panel.agent_factory.get_agents.assert_called_once() assert result == ["agent1", "agent2"] + + +def test_submit_tasks(control_panel): + """Test the submit_tasks method.""" + task1 = {"task_id": "task1"} + task2 = {"task_id": "task2"} + tasks = [task1, task2] + + # Call the method + control_panel.submit_tasks(tasks) + + # Assertions + calls = [((task1,),), ((task2,),)] + control_panel.task_mgt_strategy.add_task.assert_has_calls(calls) + assert control_panel.task_mgt_strategy.add_task.call_count == 2 + + +def test_process_tasks(control_panel): + """Test the process_tasks method.""" + # Call the method + control_panel.process_tasks() + + # Assertions + control_panel.task_mgt_strategy.process_tasks.assert_called_once_with( + control_panel.service_registry.get_services, control_panel.transport + ) + + +def test_process_tasks_exception(control_panel, caplog): + """Test process_tasks when an exception occurs.""" + # Configure mocks + control_panel.task_mgt_strategy.process_tasks.side_effect = Exception("Test error") + + # Call the method + control_panel.process_tasks() + + # Assertions + control_panel.task_mgt_strategy.process_tasks.assert_called_once_with( + control_panel.service_registry.get_services, control_panel.transport + ) + assert "Error while processing tasks: Test error" in caplog.text + + +def test_distribute_tasks(control_panel): + """Test the distribute_tasks method.""" + task = {"task_id": "task1"} + + # Call the method + control_panel.distribute_tasks(task) + + # Assertions + control_panel.task_mgt_strategy.assign_task.assert_called_once_with( + task, control_panel.service_registry.get_services + ) + + +def test_orchestrate_agents(control_panel): + """Test the orchestrate_agents method.""" + tasks = [{"task_id": "task1"}, {"task_id": "task2"}] + + # Configure mocks + control_panel.submit_tasks = MagicMock() + control_panel.process_tasks = MagicMock() + + # Call the method + control_panel.orchestrate_agents(tasks) + + # Assertions + control_panel.submit_tasks.assert_called_once_with(tasks) + control_panel.process_tasks.assert_called_once() diff --git a/pkgs/swarmauri/tests/unit/task_mgt_strategies/RoundRobinStrategy_unit_test.py b/pkgs/swarmauri/tests/unit/task_mgt_strategies/RoundRobinStrategy_unit_test.py index d28d9ab3c..c68a1a7d9 100644 --- a/pkgs/swarmauri/tests/unit/task_mgt_strategies/RoundRobinStrategy_unit_test.py +++ b/pkgs/swarmauri/tests/unit/task_mgt_strategies/RoundRobinStrategy_unit_test.py @@ -1,6 +1,6 @@ import pytest -from swarmauri.task_mgt_strategies.concrete.RoundRobinStrategy import RoundRobinStrategy from unittest.mock import MagicMock +from swarmauri.task_mgt_strategies.concrete.RoundRobinStrategy import RoundRobinStrategy @pytest.fixture @@ -14,8 +14,10 @@ def test_assign_task(round_robin_strategy): task = {"task_id": "task1", "payload": "data"} service_registry = MagicMock(return_value=["service1", "service2"]) + # Execute round_robin_strategy.assign_task(task, service_registry) + # Verify assert round_robin_strategy.task_assignments["task1"] == "service1" assert round_robin_strategy.current_index == 1 @@ -32,51 +34,67 @@ def test_assign_task_no_services(round_robin_strategy): def test_add_task(round_robin_strategy): + # Setup task = {"task_id": "task1", "payload": "data"} + # Execute round_robin_strategy.add_task(task) + # Verify assert not round_robin_strategy.task_queue.empty() queued_task = round_robin_strategy.task_queue.get() assert queued_task == task def test_remove_task(round_robin_strategy): + # Setup task_id = "task1" round_robin_strategy.task_assignments[task_id] = "service1" + # Execute round_robin_strategy.remove_task(task_id) + # Verify assert task_id not in round_robin_strategy.task_assignments def test_remove_task_not_found(round_robin_strategy): + # Setup task_id = "task1" + # Execute & Verify with pytest.raises(ValueError) as exc_info: round_robin_strategy.remove_task(task_id) assert str(exc_info.value) == "Task 'task1' not found in assignments." def test_get_task(round_robin_strategy): + # Setup task_id = "task1" round_robin_strategy.task_assignments[task_id] = "service1" + # Execute result = round_robin_strategy.get_task(task_id) - assert result == {"task_id": task_id, "assigned_service": "service1"} + # Verify + expected_result = {"task_id": task_id, "assigned_service": "service1"} + assert result == expected_result def test_get_task_not_found(round_robin_strategy): + # Setup task_id = "task1" + # Execute & Verify with pytest.raises(ValueError) as exc_info: round_robin_strategy.get_task(task_id) assert str(exc_info.value) == "Task 'task1' not found in assignments." def test_process_tasks(round_robin_strategy): + # Setup service_registry = MagicMock(return_value=["service1", "service2"]) + transport = MagicMock() tasks = [ {"task_id": "task1", "payload": "data1"}, {"task_id": "task2", "payload": "data2"}, @@ -85,9 +103,30 @@ def test_process_tasks(round_robin_strategy): for task in tasks: round_robin_strategy.add_task(task) - round_robin_strategy.process_tasks(service_registry) + # Execute + round_robin_strategy.process_tasks(service_registry, transport) + # Verify assignments assert round_robin_strategy.task_assignments["task1"] == "service1" assert round_robin_strategy.task_assignments["task2"] == "service2" assert round_robin_strategy.task_assignments["task3"] == "service1" assert round_robin_strategy.current_index == 3 + + # Verify that transport.send was called correctly + transport.send.assert_any_call(tasks[0], "service1") + transport.send.assert_any_call(tasks[1], "service2") + transport.send.assert_any_call(tasks[2], "service1") + assert transport.send.call_count == 3 + + +def test_process_tasks_no_services(round_robin_strategy): + # Setup + service_registry = MagicMock(return_value=[]) + transport = MagicMock() + task = {"task_id": "task1", "payload": "data"} + round_robin_strategy.add_task(task) + + # Execute & Verify + with pytest.raises(ValueError) as exc_info: + round_robin_strategy.process_tasks(service_registry, transport) + assert "No services available for task assignment." in str(exc_info.value) From 3a0b00d5e4bde12dfd8be82f704ca24d47e0714b Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Mon, 16 Dec 2024 13:57:42 +0100 Subject: [PATCH 082/168] feat: implement lazy loading for task management strategies and control panels --- .../swarmauri/control_panels/concrete/__init__.py | 13 +++++++++++++ .../task_mgt_strategies/concrete/__init__.py | 13 +++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 pkgs/swarmauri/swarmauri/control_panels/concrete/__init__.py diff --git a/pkgs/swarmauri/swarmauri/control_panels/concrete/__init__.py b/pkgs/swarmauri/swarmauri/control_panels/concrete/__init__.py new file mode 100644 index 000000000..767127c45 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/control_panels/concrete/__init__.py @@ -0,0 +1,13 @@ +from swarmauri.utils._lazy_import import _lazy_import + +# List of control_panels names (file names without the ".py" extension) and corresponding class names +control_panels_files = [ + ("swarmauri.control_panels.concrete.ControlPanel", "ControlPanel"), +] + +# Lazy loading of task_mgt_strategies classes, storing them in variables +for module_name, class_name in control_panels_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded state classes to __all__ +__all__ = [class_name for _, class_name in control_panels_files] diff --git a/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/__init__.py b/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/__init__.py index e69de29bb..3ee5bec23 100644 --- a/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/__init__.py @@ -0,0 +1,13 @@ +from swarmauri.utils._lazy_import import _lazy_import + +# List of task_mgt_strategies names (file names without the ".py" extension) and corresponding class names +task_mgt_strategies_files = [ + ("swarmauri.task_mgt_strategies.concrete.RoundRobinStrategy", "RoundRobinStrategy"), +] + +# Lazy loading of task_mgt_strategies classes, storing them in variables +for module_name, class_name in task_mgt_strategies_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded state classes to __all__ +__all__ = [class_name for _, class_name in task_mgt_strategies_files] From 0ce4030da667934a95b56c51180d60365dd5620f Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Mon, 16 Dec 2024 15:37:13 +0100 Subject: [PATCH 083/168] feat: extend ControlPanelBase to inherit from ComponentBase --- .../swarmauri/control_panels/base/ControlPanelBase.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py b/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py index 41653005c..2c7d443a7 100644 --- a/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py +++ b/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py @@ -1,3 +1,4 @@ +from swarmauri_core import ComponentBase from swarmauri_core.control_panels.IControlPanel import IControlPlane from typing import Any, List, Literal from pydantic import Field, ConfigDict @@ -10,7 +11,7 @@ import logging -class ControlPanelBase(IControlPlane): +class ControlPanelBase(IControlPlane, ComponentBase): """ Implementation of the ControlPlane abstract class. This class orchestrates agents, manages tasks, and ensures task distribution From 2d43f1216ed3d86b104667fd17135e3b54e5c0b5 Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Mon, 16 Dec 2024 16:52:31 +0100 Subject: [PATCH 084/168] swarm - feat: add new interfaces and restructure swarm components --- .../swarms(deprecated)/ISwarm.py | 67 +++++++++++++ .../ISwarmAgentRegistration.py | 0 .../ISwarmChainCRUD.py | 0 .../ISwarmComponent.py | 0 .../ISwarmConfigurationExporter.py | 0 .../ISwarmFactory.py | 0 .../swarms(deprecated)/__init__.py | 0 pkgs/core/swarmauri_core/swarms/ISwarm.py | 61 ++++++------ .../swarmauri/swarms(deprecated)/__init__.py | 1 + .../base/SwarmComponentBase.py | 0 .../swarms(deprecated)/base/__init__.py | 0 .../concrete/SimpleSwarmFactory.py | 0 .../swarms(deprecated)/concrete/__init__.py | 11 +++ pkgs/swarmauri/swarmauri/swarms/__init__.py | 1 - .../swarmauri/swarms/base/SwarmBase.py | 95 +++++++++++++++++++ .../swarmauri/swarms/concrete/Swarm.py | 0 .../swarmauri/swarms/concrete/__init__.py | 11 --- 17 files changed, 202 insertions(+), 45 deletions(-) create mode 100644 pkgs/core/swarmauri_core/swarms(deprecated)/ISwarm.py rename pkgs/core/swarmauri_core/{swarms => swarms(deprecated)}/ISwarmAgentRegistration.py (100%) rename pkgs/core/swarmauri_core/{swarms => swarms(deprecated)}/ISwarmChainCRUD.py (100%) rename pkgs/core/swarmauri_core/{swarms => swarms(deprecated)}/ISwarmComponent.py (100%) rename pkgs/core/swarmauri_core/{swarms => swarms(deprecated)}/ISwarmConfigurationExporter.py (100%) rename pkgs/core/swarmauri_core/{swarms => swarms(deprecated)}/ISwarmFactory.py (100%) create mode 100644 pkgs/core/swarmauri_core/swarms(deprecated)/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/swarms(deprecated)/__init__.py rename pkgs/swarmauri/swarmauri/{swarms => swarms(deprecated)}/base/SwarmComponentBase.py (100%) create mode 100644 pkgs/swarmauri/swarmauri/swarms(deprecated)/base/__init__.py rename pkgs/swarmauri/swarmauri/{swarms => swarms(deprecated)}/concrete/SimpleSwarmFactory.py (100%) create mode 100644 pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/swarms/base/SwarmBase.py create mode 100644 pkgs/swarmauri/swarmauri/swarms/concrete/Swarm.py diff --git a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarm.py b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarm.py new file mode 100644 index 000000000..7d3ecb8d8 --- /dev/null +++ b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarm.py @@ -0,0 +1,67 @@ +from abc import ABC, abstractmethod +from typing import Any, List, Dict +from datetime import datetime +from swarmauri_core.agents.IAgent import IAgent +from swarmauri_core.chains.ICallableChain import ICallableChain + +class ISwarm(ABC): + """ + Interface for a Swarm, representing a collective of agents capable of performing tasks, executing callable chains, and adaptable configurations. + """ + + # Abstract properties and setters + @property + @abstractmethod + def id(self) -> str: + """Unique identifier for the factory instance.""" + pass + + @id.setter + @abstractmethod + def id(self, value: str) -> None: + pass + + @property + @abstractmethod + def name(self) -> str: + pass + + @name.setter + @abstractmethod + def name(self, value: str) -> None: + pass + + @property + @abstractmethod + def type(self) -> str: + pass + + @type.setter + @abstractmethod + def type(self, value: str) -> None: + pass + + @property + @abstractmethod + def date_created(self) -> datetime: + pass + + @property + @abstractmethod + def last_modified(self) -> datetime: + pass + + @last_modified.setter + @abstractmethod + def last_modified(self, value: datetime) -> None: + pass + + def __hash__(self): + """ + The __hash__ method allows objects of this class to be used in sets and as dictionary keys. + __hash__ should return an integer and be defined based on immutable properties. + This is generally implemented directly in concrete classes rather than in the interface, + but it's declared here to indicate that implementing classes must provide it. + """ + pass + diff --git a/pkgs/core/swarmauri_core/swarms/ISwarmAgentRegistration.py b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmAgentRegistration.py similarity index 100% rename from pkgs/core/swarmauri_core/swarms/ISwarmAgentRegistration.py rename to pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmAgentRegistration.py diff --git a/pkgs/core/swarmauri_core/swarms/ISwarmChainCRUD.py b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmChainCRUD.py similarity index 100% rename from pkgs/core/swarmauri_core/swarms/ISwarmChainCRUD.py rename to pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmChainCRUD.py diff --git a/pkgs/core/swarmauri_core/swarms/ISwarmComponent.py b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmComponent.py similarity index 100% rename from pkgs/core/swarmauri_core/swarms/ISwarmComponent.py rename to pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmComponent.py diff --git a/pkgs/core/swarmauri_core/swarms/ISwarmConfigurationExporter.py b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmConfigurationExporter.py similarity index 100% rename from pkgs/core/swarmauri_core/swarms/ISwarmConfigurationExporter.py rename to pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmConfigurationExporter.py diff --git a/pkgs/core/swarmauri_core/swarms/ISwarmFactory.py b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmFactory.py similarity index 100% rename from pkgs/core/swarmauri_core/swarms/ISwarmFactory.py rename to pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmFactory.py diff --git a/pkgs/core/swarmauri_core/swarms(deprecated)/__init__.py b/pkgs/core/swarmauri_core/swarms(deprecated)/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/core/swarmauri_core/swarms/ISwarm.py b/pkgs/core/swarmauri_core/swarms/ISwarm.py index 7d3ecb8d8..12ceba54c 100644 --- a/pkgs/core/swarmauri_core/swarms/ISwarm.py +++ b/pkgs/core/swarmauri_core/swarms/ISwarm.py @@ -1,67 +1,62 @@ from abc import ABC, abstractmethod -from typing import Any, List, Dict -from datetime import datetime -from swarmauri_core.agents.IAgent import IAgent -from swarmauri_core.chains.ICallableChain import ICallableChain +from typing import Any, List, Tuple + class ISwarm(ABC): - """ - Interface for a Swarm, representing a collective of agents capable of performing tasks, executing callable chains, and adaptable configurations. - """ + """Abstract interface for swarm implementations with both sync/async operations.""" - # Abstract properties and setters @property @abstractmethod - def id(self) -> str: - """Unique identifier for the factory instance.""" + def num_agents(self) -> int: + """Number of agents in the swarm.""" pass - @id.setter @abstractmethod - def id(self, value: str) -> None: + def add_agent(self, agent: Any) -> None: + """Add an agent to the swarm.""" pass - @property @abstractmethod - def name(self) -> str: + def remove_agent(self, agent_id: int) -> None: + """Remove an agent from the swarm.""" pass - @name.setter @abstractmethod - def name(self, value: str) -> None: + def replace_agent(self, agent_id: int, new_agent: Any) -> None: + """Replace an agent in the swarm.""" pass - @property @abstractmethod - def type(self) -> str: + def get_agent_statuses(self) -> List[Tuple[int, Any]]: + """Get status of all agents.""" pass - @type.setter @abstractmethod - def type(self, value: str) -> None: + def distribute_task(self, task: Any) -> None: + """Distribute a task to the swarm.""" pass - @property @abstractmethod - def date_created(self) -> datetime: + def collect_results(self) -> List[Any]: + """Collect results from the swarm.""" pass - @property @abstractmethod - def last_modified(self) -> datetime: + def run(self, tasks: List[Any]) -> List[Any]: + """Execute tasks synchronously.""" pass - @last_modified.setter @abstractmethod - def last_modified(self, value: datetime) -> None: + async def arun(self, tasks: List[Any]) -> List[Any]: + """Execute tasks asynchronously.""" pass - def __hash__(self): - """ - The __hash__ method allows objects of this class to be used in sets and as dictionary keys. - __hash__ should return an integer and be defined based on immutable properties. - This is generally implemented directly in concrete classes rather than in the interface, - but it's declared here to indicate that implementing classes must provide it. - """ + @abstractmethod + def process_task(self, task: Any) -> Any: + """Process a single task synchronously.""" pass + @abstractmethod + async def aprocess_task(self, task: Any) -> Any: + """Process a single task asynchronously.""" + pass diff --git a/pkgs/swarmauri/swarmauri/swarms(deprecated)/__init__.py b/pkgs/swarmauri/swarmauri/swarms(deprecated)/__init__.py new file mode 100644 index 000000000..97c140f08 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/swarms(deprecated)/__init__.py @@ -0,0 +1 @@ +from swarmauri.swarms.concrete import * diff --git a/pkgs/swarmauri/swarmauri/swarms/base/SwarmComponentBase.py b/pkgs/swarmauri/swarmauri/swarms(deprecated)/base/SwarmComponentBase.py similarity index 100% rename from pkgs/swarmauri/swarmauri/swarms/base/SwarmComponentBase.py rename to pkgs/swarmauri/swarmauri/swarms(deprecated)/base/SwarmComponentBase.py diff --git a/pkgs/swarmauri/swarmauri/swarms(deprecated)/base/__init__.py b/pkgs/swarmauri/swarmauri/swarms(deprecated)/base/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/swarms/concrete/SimpleSwarmFactory.py b/pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/SimpleSwarmFactory.py similarity index 100% rename from pkgs/swarmauri/swarmauri/swarms/concrete/SimpleSwarmFactory.py rename to pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/SimpleSwarmFactory.py diff --git a/pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/__init__.py b/pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/__init__.py new file mode 100644 index 000000000..61f84eae6 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/__init__.py @@ -0,0 +1,11 @@ +from swarmauri.utils._lazy_import import _lazy_import + +# List of swarms names (file names without the ".py" extension) and corresponding class names +swarms_files = [("swarmauri.swarms.concrete.SimpleSwarmFactory", "SimpleSwarmFactory")] + +# Lazy loading of swarms classes, storing them in variables +for module_name, class_name in swarms_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded swarms classes to __all__ +__all__ = [class_name for _, class_name in swarms_files] diff --git a/pkgs/swarmauri/swarmauri/swarms/__init__.py b/pkgs/swarmauri/swarmauri/swarms/__init__.py index 97c140f08..e69de29bb 100644 --- a/pkgs/swarmauri/swarmauri/swarms/__init__.py +++ b/pkgs/swarmauri/swarmauri/swarms/__init__.py @@ -1 +0,0 @@ -from swarmauri.swarms.concrete import * diff --git a/pkgs/swarmauri/swarmauri/swarms/base/SwarmBase.py b/pkgs/swarmauri/swarmauri/swarms/base/SwarmBase.py new file mode 100644 index 000000000..8ac192822 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/swarms/base/SwarmBase.py @@ -0,0 +1,95 @@ +import asyncio +from typing import Any, List, Tuple +from swarmauri_core.swarms.ISwarm import ISwarm + + +class SwarmBase(ISwarm): + def __init__( + self, num_agents: int = 5, agent_timeout: float = 1.0, retry_attempts: int = 3 + ): + self._agents: List[Any] = [] + self._num_agents = num_agents + self._agent_timeout = agent_timeout + self._retry_attempts = retry_attempts + self._task_queue: asyncio.Queue = asyncio.Queue() + self._results_queue: asyncio.Queue = asyncio.Queue() + + @property + def num_agents(self) -> int: + return self._num_agents + + def add_agent(self, agent: SwarmBaseAgent) -> None: + self._agents.append(agent) + self._num_agents = len(self._agents) + + def remove_agent(self, agent_id: int) -> None: + if 0 <= agent_id < len(self._agents): + self._agents.pop(agent_id) + self._num_agents = len(self._agents) + + def replace_agent(self, agent_id: int, new_agent: SwarmBaseAgent) -> None: + if 0 <= agent_id < len(self._agents): + self._agents[agent_id] = new_agent + + def get_agent_statuses(self) -> List[Tuple[int, Any]]: + return [(i, agent.status) for i, agent in enumerate(self._agents)] + + def distribute_task(self, task: Any) -> None: + self._task_queue.put_nowait(task) + + def collect_results(self) -> List[Any]: + results = [] + while not self._results_queue.empty(): + results.append(self._results_queue.get_nowait()) + return results + + def run(self, tasks: List[Any]) -> List[Any]: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + return loop.run_until_complete(self.arun(tasks)) + finally: + loop.close() + + async def arun(self, tasks: List[Any]) -> List[Any]: + for task in tasks: + await self._task_queue.put(task) + + agent_tasks = [ + asyncio.create_task(self._process_worker(agent)) for agent in self._agents + ] + await asyncio.gather(*agent_tasks) + return await self._collect_results_async() + + def process_task(self, task: Any) -> Any: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + return loop.run_until_complete(self.aprocess_task(task)) + finally: + loop.close() + + async def aprocess_task(self, task: Any) -> Any: + if not self._agents: + raise RuntimeError("No agents available to process task") + return await self._agents[0].process(task) + + async def _process_worker(self, agent: SwarmBaseAgent) -> None: + retry_count = 0 + while retry_count < self._retry_attempts: + try: + task = await asyncio.wait_for( + self._task_queue.get(), timeout=self._agent_timeout + ) + result = await agent.process(task) + await self._results_queue.put(result) + self._task_queue.task_done() + break + except Exception: + retry_count += 1 + + async def _collect_results_async(self) -> List[Any]: + results = [] + while not self._results_queue.empty(): + results.append(await self._results_queue.get()) + return results diff --git a/pkgs/swarmauri/swarmauri/swarms/concrete/Swarm.py b/pkgs/swarmauri/swarmauri/swarms/concrete/Swarm.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/swarms/concrete/__init__.py b/pkgs/swarmauri/swarmauri/swarms/concrete/__init__.py index 61f84eae6..e69de29bb 100644 --- a/pkgs/swarmauri/swarmauri/swarms/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/swarms/concrete/__init__.py @@ -1,11 +0,0 @@ -from swarmauri.utils._lazy_import import _lazy_import - -# List of swarms names (file names without the ".py" extension) and corresponding class names -swarms_files = [("swarmauri.swarms.concrete.SimpleSwarmFactory", "SimpleSwarmFactory")] - -# Lazy loading of swarms classes, storing them in variables -for module_name, class_name in swarms_files: - globals()[class_name] = _lazy_import(module_name, class_name) - -# Adding the lazy-loaded swarms classes to __all__ -__all__ = [class_name for _, class_name in swarms_files] From f6b28da7f68eaa09570ff816ee1c8a3b4d8ce8ba Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Tue, 17 Dec 2024 14:01:56 +0100 Subject: [PATCH 085/168] feat: implement Swarm class with task execution and error handling, add lazy loading for swarm classes --- pkgs/core/swarmauri_core/swarms/ISwarm.py | 62 ++----- .../swarmauri/swarms/base/SwarmBase.py | 160 +++++++++--------- .../swarmauri/swarms/concrete/Swarm.py | 36 ++++ .../swarmauri/swarms/concrete/__init__.py | 13 ++ .../tests/unit/swarms/Swarm_unit_test.py | 91 ++++++++++ 5 files changed, 239 insertions(+), 123 deletions(-) create mode 100644 pkgs/swarmauri/tests/unit/swarms/Swarm_unit_test.py diff --git a/pkgs/core/swarmauri_core/swarms/ISwarm.py b/pkgs/core/swarmauri_core/swarms/ISwarm.py index 12ceba54c..7f3e0bb15 100644 --- a/pkgs/core/swarmauri_core/swarms/ISwarm.py +++ b/pkgs/core/swarmauri_core/swarms/ISwarm.py @@ -1,62 +1,32 @@ from abc import ABC, abstractmethod -from typing import Any, List, Tuple +from typing import Any, Dict, List, Optional, Union class ISwarm(ABC): - """Abstract interface for swarm implementations with both sync/async operations.""" - - @property - @abstractmethod - def num_agents(self) -> int: - """Number of agents in the swarm.""" - pass - - @abstractmethod - def add_agent(self, agent: Any) -> None: - """Add an agent to the swarm.""" - pass - - @abstractmethod - def remove_agent(self, agent_id: int) -> None: - """Remove an agent from the swarm.""" - pass - - @abstractmethod - def replace_agent(self, agent_id: int, new_agent: Any) -> None: - """Replace an agent in the swarm.""" - pass - - @abstractmethod - def get_agent_statuses(self) -> List[Tuple[int, Any]]: - """Get status of all agents.""" - pass - - @abstractmethod - def distribute_task(self, task: Any) -> None: - """Distribute a task to the swarm.""" - pass - - @abstractmethod - def collect_results(self) -> List[Any]: - """Collect results from the swarm.""" - pass + """Abstract base class for swarm implementations""" @abstractmethod - def run(self, tasks: List[Any]) -> List[Any]: - """Execute tasks synchronously.""" + async def exec( + self, + input_data: Union[str, List[str]], + **kwargs: Dict[str, Any], + ) -> Any: + """Execute swarm tasks with given input""" pass @abstractmethod - async def arun(self, tasks: List[Any]) -> List[Any]: - """Execute tasks asynchronously.""" + def get_swarm_status(self) -> Dict[int, Any]: + """Get status of all agents in the swarm""" pass + @property @abstractmethod - def process_task(self, task: Any) -> Any: - """Process a single task synchronously.""" + def agents(self) -> List[Any]: + """Get list of agents in the swarm""" pass + @property @abstractmethod - async def aprocess_task(self, task: Any) -> Any: - """Process a single task asynchronously.""" + def queue_size(self) -> int: + """Get size of task queue""" pass diff --git a/pkgs/swarmauri/swarmauri/swarms/base/SwarmBase.py b/pkgs/swarmauri/swarmauri/swarms/base/SwarmBase.py index 8ac192822..45972f930 100644 --- a/pkgs/swarmauri/swarmauri/swarms/base/SwarmBase.py +++ b/pkgs/swarmauri/swarmauri/swarms/base/SwarmBase.py @@ -1,95 +1,101 @@ import asyncio -from typing import Any, List, Tuple +from typing import Any, Dict, List, Literal, Optional, Union +from pydantic import ConfigDict, Field +from enum import Enum +from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes from swarmauri_core.swarms.ISwarm import ISwarm -class SwarmBase(ISwarm): - def __init__( - self, num_agents: int = 5, agent_timeout: float = 1.0, retry_attempts: int = 3 - ): - self._agents: List[Any] = [] - self._num_agents = num_agents - self._agent_timeout = agent_timeout - self._retry_attempts = retry_attempts - self._task_queue: asyncio.Queue = asyncio.Queue() - self._results_queue: asyncio.Queue = asyncio.Queue() +class SwarmStatus(Enum): + IDLE = "IDLE" + WORKING = "WORKING" + COMPLETED = "COMPLETED" + FAILED = "FAILED" - @property - def num_agents(self) -> int: - return self._num_agents - def add_agent(self, agent: SwarmBaseAgent) -> None: - self._agents.append(agent) - self._num_agents = len(self._agents) +class SwarmBase(ISwarm, ComponentBase): + """Base class for Swarm implementations""" - def remove_agent(self, agent_id: int) -> None: - if 0 <= agent_id < len(self._agents): - self._agents.pop(agent_id) - self._num_agents = len(self._agents) + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + resource: Optional[str] = Field(default=ResourceTypes.SWARM.value, frozen=True) + type: Literal["SwarmBase"] = "SwarmBase" - def replace_agent(self, agent_id: int, new_agent: SwarmBaseAgent) -> None: - if 0 <= agent_id < len(self._agents): - self._agents[agent_id] = new_agent + num_agents: int = Field(default=5, gt=0, le=100) + agent_timeout: float = Field(default=1.0, gt=0) + max_retries: int = Field(default=3, ge=0) + max_queue_size: int = Field(default=10, gt=0) - def get_agent_statuses(self) -> List[Tuple[int, Any]]: - return [(i, agent.status) for i, agent in enumerate(self._agents)] + _agents: List[Any] = [] + _task_queue: Optional[asyncio.Queue] = None + _status: Dict[int, SwarmStatus] = {} - def distribute_task(self, task: Any) -> None: - self._task_queue.put_nowait(task) + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._task_queue = asyncio.Queue(maxsize=self.max_queue_size) + self._initialize_agents() - def collect_results(self) -> List[Any]: - results = [] - while not self._results_queue.empty(): - results.append(self._results_queue.get_nowait()) - return results + def _initialize_agents(self): + self._agents = [self._create_agent() for _ in range(self.num_agents)] + self._status = {i: SwarmStatus.IDLE for i in range(self.num_agents)} - def run(self, tasks: List[Any]) -> List[Any]: - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - try: - return loop.run_until_complete(self.arun(tasks)) - finally: - loop.close() + def _create_agent(self) -> Any: + """create specific agent types""" + raise NotImplementedError("Agent creation method not implemented") - async def arun(self, tasks: List[Any]) -> List[Any]: - for task in tasks: - await self._task_queue.put(task) + @property + def agents(self) -> List[Any]: + return self._agents + + @property + def queue_size(self) -> int: + return self._task_queue.qsize() - agent_tasks = [ - asyncio.create_task(self._process_worker(agent)) for agent in self._agents - ] - await asyncio.gather(*agent_tasks) - return await self._collect_results_async() - def process_task(self, task: Any) -> Any: - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + def get_swarm_status(self) -> Dict[int, SwarmStatus]: + return self._status + + async def _process_task(self, agent_id: int, task: Any, **kwargs) -> Any: + self._status[agent_id] = SwarmStatus.WORKING try: - return loop.run_until_complete(self.aprocess_task(task)) - finally: - loop.close() - - async def aprocess_task(self, task: Any) -> Any: - if not self._agents: - raise RuntimeError("No agents available to process task") - return await self._agents[0].process(task) - - async def _process_worker(self, agent: SwarmBaseAgent) -> None: - retry_count = 0 - while retry_count < self._retry_attempts: - try: - task = await asyncio.wait_for( - self._task_queue.get(), timeout=self._agent_timeout - ) - result = await agent.process(task) - await self._results_queue.put(result) - self._task_queue.task_done() - break - except Exception: - retry_count += 1 - - async def _collect_results_async(self) -> List[Any]: + for _ in range(self.max_retries): + try: + result = await asyncio.wait_for( + self._execute_task(task, agent_id, **kwargs), timeout=self.agent_timeout + ) + self._status[agent_id] = SwarmStatus.COMPLETED + return result + except asyncio.TimeoutError: + continue + self._status[agent_id] = SwarmStatus.FAILED + return None + except Exception as e: + self._status[agent_id] = SwarmStatus.FAILED + raise e + + async def _execute_task(self, task: Any, agent_id: int) -> Any: + """Override this method to implement specific task execution logic""" + raise NotImplementedError("Task execution method not implemented") + + async def exec( + self, input_data: Union[List[str], Any] = [], **kwargs: Optional[Dict] + ) -> List[Any]: + tasks = input_data if isinstance(input_data, list) else [input_data] + for task in tasks: + await self._task_queue.put(task) + results = [] - while not self._results_queue.empty(): - results.append(await self._results_queue.get()) + while not self._task_queue.empty(): + available_agents = [ + i for i, status in self._status.items() if status == SwarmStatus.IDLE + ] + if not available_agents: + await asyncio.sleep(0.1) + continue + + task = await self._task_queue.get() + agent_id = available_agents[0] + result = await self._process_task(agent_id, task, **kwargs) + if result is not None: + results.append(result) + return results diff --git a/pkgs/swarmauri/swarmauri/swarms/concrete/Swarm.py b/pkgs/swarmauri/swarmauri/swarms/concrete/Swarm.py index e69de29bb..2c4c7fee8 100644 --- a/pkgs/swarmauri/swarmauri/swarms/concrete/Swarm.py +++ b/pkgs/swarmauri/swarmauri/swarms/concrete/Swarm.py @@ -0,0 +1,36 @@ +from typing import Any, Dict, List, Literal, Optional, Type, Union +from pydantic import Field + +from swarmauri.swarms.base.SwarmBase import SwarmBase, SwarmStatus + + +class Swarm(SwarmBase): + """Concrete implementation of SwarmBase for task processing""" + + type: Literal["Swarm"] = "Swarm" + agent_class: Type[Any] = Field(description="Agent class to use for swarm") + task_batch_size: int = Field(default=1, gt=0) + + def _create_agent(self) -> Any: + """Create new agent instance""" + return self.agent_class() + + async def _execute_task(self, task: Any, agent_id: int, **kwargs) -> Dict[str, Any]: + """Execute task using specified agent""" + agent = self._agents[agent_id] + try: + result = await agent.process(task, **kwargs) + return { + "agent_id": agent_id, + "status": SwarmStatus.COMPLETED, + "result": result, + } + except Exception as e: + return {"agent_id": agent_id, "status": SwarmStatus.FAILED, "error": str(e)} + + async def exec( + self, input_data: Union[List[str], Any] = [], **kwargs: Optional[Dict] + ) -> List[Dict[str, Any]]: + """Execute tasks in parallel using available agents""" + results = await super().exec(input_data, **kwargs) + return results diff --git a/pkgs/swarmauri/swarmauri/swarms/concrete/__init__.py b/pkgs/swarmauri/swarmauri/swarms/concrete/__init__.py index e69de29bb..98af9eb31 100644 --- a/pkgs/swarmauri/swarmauri/swarms/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/swarms/concrete/__init__.py @@ -0,0 +1,13 @@ +from swarmauri.utils._lazy_import import _lazy_import + +# List of swarms names (file names without the ".py" extension) and corresponding class names +swarms_files = [ + ("swarmauri.swarms.concrete.Swarm", "Swarm") +] + +# Lazy loading of swarms classes, storing them in variables +for module_name, class_name in swarms_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded swarms classes to __all__ +__all__ = [class_name for _, class_name in swarms_files] diff --git a/pkgs/swarmauri/tests/unit/swarms/Swarm_unit_test.py b/pkgs/swarmauri/tests/unit/swarms/Swarm_unit_test.py new file mode 100644 index 000000000..966ee48c9 --- /dev/null +++ b/pkgs/swarmauri/tests/unit/swarms/Swarm_unit_test.py @@ -0,0 +1,91 @@ +from typing import Any +from pydantic import BaseModel +import pytest +import asyncio +from swarmauri.swarms.base.SwarmBase import SwarmStatus +from swarmauri.swarms.concrete.Swarm import Swarm + + +class MockAgent(BaseModel): + async def process(self, task: Any, **kwargs) -> str: + if task == "fail": + raise Exception("Task failed") + return f"Processed {task}" + +@pytest.fixture +def swarm(): + return Swarm(agent_class=MockAgent, num_agents=3, agent_timeout=0.5, max_retries=2) + +@pytest.mark.unit +def test_ubc_resource(swarm): + assert swarm.resource == "Swarm" + + +@pytest.mark.unit +def test_ubc_type(swarm): + assert swarm.type == "Swarm" + + +@pytest.mark.unit +def test_serialization(swarm): + assert swarm.id == Swarm.model_validate_json(swarm.model_dump_json()).id + + +@pytest.mark.asyncio +async def test_swarm_initialization(swarm): + assert len(swarm.agents) == 3 + assert swarm.queue_size == 0 + assert all(s == SwarmStatus.IDLE for s in swarm.get_swarm_status().values()) + + +@pytest.mark.asyncio +async def test_single_task_execution(swarm): + results = await swarm.exec("task1") + assert len(results) == 1 + assert results[0]["status"] == SwarmStatus.COMPLETED + assert results[0]["result"] == "Processed task1" + assert "agent_id" in results[0] + + +@pytest.mark.asyncio +async def test_multiple_tasks_execution(swarm): + tasks = ["task1", "task2", "task3"] + results = await swarm.exec(tasks) + assert len(results) == 3 + assert all(r["status"] == SwarmStatus.COMPLETED for r in results) + assert all("Processed" in r["result"] for r in results) + + +@pytest.mark.asyncio +async def test_failed_task_handling(swarm): + results = await swarm.exec("fail") + assert len(results) == 1 + assert results[0]["status"] == SwarmStatus.FAILED + assert "error" in results[0] + + +@pytest.mark.asyncio(loop_scope="session") +async def test_swarm_status_changes(swarm): + # Create tasks + tasks = ["task1"] * 3 + + # Start execution + task_future = asyncio.create_task(swarm.exec(tasks)) + + # Wait briefly for tasks to start + await asyncio.sleep(0.1) + + # Check intermediate status + status = swarm.get_swarm_status() + assert any( + s in [SwarmStatus.WORKING, SwarmStatus.COMPLETED] for s in status.values() + ) + + # Wait for completion with timeout + try: + results = await asyncio.wait_for(task_future, timeout=2.0) + assert len(results) == len(tasks) + assert all(r["status"] == SwarmStatus.COMPLETED for r in results) + except asyncio.TimeoutError: + task_future.cancel() + raise From 6642ac155befdf2e1da70f04cdb0261d2fa0db56 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 07:13:41 -0600 Subject: [PATCH 086/168] Delete pkgs/core/swarmauri_core/swarms(deprecated)/ISwarm.py --- .../swarms(deprecated)/ISwarm.py | 67 ------------------- 1 file changed, 67 deletions(-) delete mode 100644 pkgs/core/swarmauri_core/swarms(deprecated)/ISwarm.py diff --git a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarm.py b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarm.py deleted file mode 100644 index 7d3ecb8d8..000000000 --- a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarm.py +++ /dev/null @@ -1,67 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Any, List, Dict -from datetime import datetime -from swarmauri_core.agents.IAgent import IAgent -from swarmauri_core.chains.ICallableChain import ICallableChain - -class ISwarm(ABC): - """ - Interface for a Swarm, representing a collective of agents capable of performing tasks, executing callable chains, and adaptable configurations. - """ - - # Abstract properties and setters - @property - @abstractmethod - def id(self) -> str: - """Unique identifier for the factory instance.""" - pass - - @id.setter - @abstractmethod - def id(self, value: str) -> None: - pass - - @property - @abstractmethod - def name(self) -> str: - pass - - @name.setter - @abstractmethod - def name(self, value: str) -> None: - pass - - @property - @abstractmethod - def type(self) -> str: - pass - - @type.setter - @abstractmethod - def type(self, value: str) -> None: - pass - - @property - @abstractmethod - def date_created(self) -> datetime: - pass - - @property - @abstractmethod - def last_modified(self) -> datetime: - pass - - @last_modified.setter - @abstractmethod - def last_modified(self, value: datetime) -> None: - pass - - def __hash__(self): - """ - The __hash__ method allows objects of this class to be used in sets and as dictionary keys. - __hash__ should return an integer and be defined based on immutable properties. - This is generally implemented directly in concrete classes rather than in the interface, - but it's declared here to indicate that implementing classes must provide it. - """ - pass - From 5d3979727362fd1a55e4f6d8358499b5f3969afe Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Tue, 17 Dec 2024 14:49:45 +0100 Subject: [PATCH 087/168] refactor: remove deprecated swarm components and interfaces --- .../ISwarmAgentRegistration.py | 73 ----------- .../swarms(deprecated)/ISwarmChainCRUD.py | 62 --------- .../swarms(deprecated)/ISwarmComponent.py | 13 -- .../ISwarmConfigurationExporter.py | 33 ----- .../swarms(deprecated)/ISwarmFactory.py | 120 ------------------ .../swarms(deprecated)/__init__.py | 0 .../swarmauri/swarms(deprecated)/__init__.py | 1 - .../base/SwarmComponentBase.py | 15 --- .../swarms(deprecated)/base/__init__.py | 0 .../concrete/SimpleSwarmFactory.py | 50 -------- .../swarms(deprecated)/concrete/__init__.py | 11 -- .../swarmauri/swarms/base/SwarmBase.py | 4 +- 12 files changed, 2 insertions(+), 380 deletions(-) delete mode 100644 pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmAgentRegistration.py delete mode 100644 pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmChainCRUD.py delete mode 100644 pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmComponent.py delete mode 100644 pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmConfigurationExporter.py delete mode 100644 pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmFactory.py delete mode 100644 pkgs/core/swarmauri_core/swarms(deprecated)/__init__.py delete mode 100644 pkgs/swarmauri/swarmauri/swarms(deprecated)/__init__.py delete mode 100644 pkgs/swarmauri/swarmauri/swarms(deprecated)/base/SwarmComponentBase.py delete mode 100644 pkgs/swarmauri/swarmauri/swarms(deprecated)/base/__init__.py delete mode 100644 pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/SimpleSwarmFactory.py delete mode 100644 pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/__init__.py diff --git a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmAgentRegistration.py b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmAgentRegistration.py deleted file mode 100644 index a32dc235d..000000000 --- a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmAgentRegistration.py +++ /dev/null @@ -1,73 +0,0 @@ -from abc import ABC, abstractmethod -from typing import List, Dict, Optional -from swarmauri_core.agents.IAgent import IAgent - -class ISwarmAgentRegistration(ABC): - """ - Interface for registering agents with the swarm, designed to support CRUD operations on IAgent instances. - """ - - @id.setter - @abstractmethod - def registry(self, value: str) -> None: - pass - - @property - @abstractmethod - def registry(self) -> List[IAgent]: - pass - - @abstractmethod - def register_agent(self, agent: IAgent) -> bool: - """ - Register a new agent with the swarm. - - Parameters: - agent (IAgent): An instance of IAgent representing the agent to register. - - Returns: - bool: True if the registration succeeded; False otherwise. - """ - pass - - @abstractmethod - def update_agent(self, agent_id: str, updated_agent: IAgent) -> bool: - """ - Update the details of an existing agent. This could include changing the agent's configuration, - task assignment, or any other mutable attribute. - - Parameters: - agent_id (str): The unique identifier for the agent. - updated_agent (IAgent): An updated IAgent instance to replace the existing one. - - Returns: - bool: True if the update was successful; False otherwise. - """ - pass - - @abstractmethod - def remove_agent(self, agent_id: str) -> bool: - """ - Remove an agent from the swarm based on its unique identifier. - - Parameters: - agent_id (str): The unique identifier for the agent to be removed. - - Returns: - bool: True if the removal was successful; False otherwise. - """ - pass - - @abstractmethod - def get_agent(self, agent_id: str) -> Optional[IAgent]: - """ - Retrieve an agent's instance from its unique identifier. - - Parameters: - agent_id (str): The unique identifier for the agent of interest. - - Returns: - Optional[IAgent]: The IAgent instance if found; None otherwise. - """ - pass - diff --git a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmChainCRUD.py b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmChainCRUD.py deleted file mode 100644 index dcb0cf191..000000000 --- a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmChainCRUD.py +++ /dev/null @@ -1,62 +0,0 @@ -from abc import ABC, abstractmethod -from typing import List, Dict, Any - -class ISwarmChainCRUD(ABC): - """ - Interface to provide CRUD operations for ICallableChain within swarms. - """ - - @abstractmethod - def create_chain(self, chain_id: str, chain_definition: Dict[str, Any]) -> None: - """ - Creates a callable chain with the provided definition. - - Parameters: - - chain_id (str): A unique identifier for the callable chain. - - chain_definition (Dict[str, Any]): The definition of the callable chain including steps and their configurations. - """ - pass - - @abstractmethod - def read_chain(self, chain_id: str) -> Dict[str, Any]: - """ - Retrieves the definition of a callable chain by its identifier. - - Parameters: - - chain_id (str): The unique identifier of the callable chain to be retrieved. - - Returns: - - Dict[str, Any]: The definition of the callable chain. - """ - pass - - @abstractmethod - def update_chain(self, chain_id: str, new_definition: Dict[str, Any]) -> None: - """ - Updates an existing callable chain with a new definition. - - Parameters: - - chain_id (str): The unique identifier of the callable chain to be updated. - - new_definition (Dict[str, Any]): The new definition of the callable chain including updated steps and configurations. - """ - pass - - @abstractmethod - def delete_chain(self, chain_id: str) -> None: - """ - Removes a callable chain from the swarm. - - Parameters: - - chain_id (str): The unique identifier of the callable chain to be removed. - """ - pass - - @abstractmethod - def list_chains(self) -> List[Dict[str, Any]]: - """ - Lists all callable chains currently managed by the swarm. - - Returns: - - List[Dict[str, Any]]: A list of callable chain definitions. - """ - pass \ No newline at end of file diff --git a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmComponent.py b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmComponent.py deleted file mode 100644 index aee78b027..000000000 --- a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmComponent.py +++ /dev/null @@ -1,13 +0,0 @@ -from abc import ABC, abstractmethod - -class ISwarmComponent(ABC): - """ - Interface for defining a general component within a swarm system. - """ - - @abstractmethod - def __init__(self, key: str, name: str): - """ - Initializes a swarm component with a unique key and name. - """ - pass \ No newline at end of file diff --git a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmConfigurationExporter.py b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmConfigurationExporter.py deleted file mode 100644 index 3b1672ca3..000000000 --- a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmConfigurationExporter.py +++ /dev/null @@ -1,33 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Dict -class ISwarmConfigurationExporter(ABC): - - @abstractmethod - def to_dict(self) -> Dict: - """ - Serializes the swarm configuration to a dictionary. - - Returns: - Dict: The serialized configuration as a dictionary. - """ - pass - - @abstractmethod - def to_json(self) -> str: - """ - Serializes the swarm configuration to a JSON string. - - Returns: - str: The serialized configuration as a JSON string. - """ - pass - - @abstractmethod - def to_pickle(self) -> bytes: - """ - Serializes the swarm configuration to a Pickle byte stream. - - Returns: - bytes: The serialized configuration as a Pickle byte stream. - """ - pass \ No newline at end of file diff --git a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmFactory.py b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmFactory.py deleted file mode 100644 index d26416114..000000000 --- a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmFactory.py +++ /dev/null @@ -1,120 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, List, NamedTuple, Optional, Type, Union -from swarmauri_core.swarms.ISwarm import ISwarm -from swarmauri_core.chains.ICallableChain import ICallableChain -from swarmauri_core.agents.IAgent import IAgent - -class Step(NamedTuple): - description: str - callable: Callable # Reference to the function to execute - args: Optional[List[Any]] = None - kwargs: Optional[Dict[str, Any]] = None - -class CallableChainItem(NamedTuple): - key: str # Unique identifier for the item within the chain - execution_context: Dict[str, Any] # Execution context and metadata - steps: List[Step] - -class AgentDefinition(NamedTuple): - type: str - configuration: Dict[str, Any] - capabilities: List[str] - dependencies: List[str] - execution_context: Dict[str, Any] - -class FunctionParameter(NamedTuple): - name: str - type: Type - default: Optional[Any] = None - required: bool = True - -class FunctionDefinition(NamedTuple): - identifier: str - parameters: List[FunctionParameter] - return_type: Type - execution_context: Dict[str, Any] - callable_source: Callable - -class ISwarmFactory(ABC): - - @abstractmethod - def create_swarm(self, *args, **kwargs) -> ISwarm: - """ - Creates and returns a new swarm instance configured with the provided arguments. - """ - pass - - @abstractmethod - def create_agent(self, agent_definition: AgentDefinition) -> IAgent: - """ - Creates a new agent based on the provided enhanced agent definition. - - Args: - agent_definition: An instance of AgentDefinition detailing the agent's setup. - - Returns: - An instance or identifier of the newly created agent. - """ - pass - - @abstractmethod - def create_callable_chain(self, chain_definition: List[CallableChainItem]) -> ICallableChain: - """ - Creates a new callable chain based on the provided definition. - - Args: - chain_definition: Details required to build the chain, such as sequence of functions and arguments. - - Returns: - ICallableChain: The constructed callable chain instance. - """ - pass - - @abstractmethod - def register_function(self, function_definition: FunctionDefinition) -> None: - """ - Registers a function within the factory ecosystem, making it available for callable chains and agents. - - Args: - function_definition: An instance of FunctionDefinition detailing the function's specification. - """ - pass - - @abstractmethod - def export_callable_chains(self, format_type: str = 'json') -> Union[dict, str, bytes]: - """ - Exports configurations of all callable chains in the specified format. - Supported formats: 'json', 'pickle'. - - Args: - format_type (str): The format for exporting the configurations. - - Returns: - Union[dict, str, bytes]: The callable chain configurations in the specified format. - """ - pass - - @abstractmethod - def load_callable_chains(self, chains_data, format_type: str = 'json'): - """ - Loads callable chain configurations from given data. - - Args: - chains_data (Union[dict, str, bytes]): Data containing callable chain configurations. - format_type (str): The format of the provided chains data. - """ - pass - - @abstractmethod - def export_configuration(self, format_type: str = 'json') -> Union[dict, str, bytes]: - """ - Exports the swarm's and agents' configurations in the specified format. - Supported formats: 'json', 'pickle'. Default is 'json'. - - Args: - format_type (str): The format for exporting the configurations. - - Returns: - Union[dict, str, bytes]: The configurations in the specified format. - """ - pass diff --git a/pkgs/core/swarmauri_core/swarms(deprecated)/__init__.py b/pkgs/core/swarmauri_core/swarms(deprecated)/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/pkgs/swarmauri/swarmauri/swarms(deprecated)/__init__.py b/pkgs/swarmauri/swarmauri/swarms(deprecated)/__init__.py deleted file mode 100644 index 97c140f08..000000000 --- a/pkgs/swarmauri/swarmauri/swarms(deprecated)/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from swarmauri.swarms.concrete import * diff --git a/pkgs/swarmauri/swarmauri/swarms(deprecated)/base/SwarmComponentBase.py b/pkgs/swarmauri/swarmauri/swarms(deprecated)/base/SwarmComponentBase.py deleted file mode 100644 index 8e643494a..000000000 --- a/pkgs/swarmauri/swarmauri/swarms(deprecated)/base/SwarmComponentBase.py +++ /dev/null @@ -1,15 +0,0 @@ -from swarmauri_core.swarms.ISwarmComponent import ISwarmComponent - -class SwarmComponentBase(ISwarmComponent): - """ - Interface for defining basics of any component within the swarm system. - """ - def __init__(self, key: str, name: str, superclass: str, module: str, class_name: str, args=None, kwargs=None): - self.key = key - self.name = name - self.superclass = superclass - self.module = module - self.class_name = class_name - self.args = args or [] - self.kwargs = kwargs or {} - \ No newline at end of file diff --git a/pkgs/swarmauri/swarmauri/swarms(deprecated)/base/__init__.py b/pkgs/swarmauri/swarmauri/swarms(deprecated)/base/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/SimpleSwarmFactory.py b/pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/SimpleSwarmFactory.py deleted file mode 100644 index beaec4e49..000000000 --- a/pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/SimpleSwarmFactory.py +++ /dev/null @@ -1,50 +0,0 @@ -import json -import pickle -from typing import List -from swarmauri_core.chains.ISwarmFactory import ( - ISwarmFactory , - CallableChainItem, - AgentDefinition, - FunctionDefinition -) -class SimpleSwarmFactory(ISwarmFactory): - def __init__(self): - self.swarms = [] - self.callable_chains = [] - - def create_swarm(self, agents=[]): - swarm = {"agents": agents} - self.swarms.append(swarm) - return swarm - - def create_agent(self, agent_definition: AgentDefinition): - # For simplicity, agents are stored in a list - # Real-world usage might involve more sophisticated management and instantiation based on type and configuration - agent = {"definition": agent_definition._asdict()} - self.agents.append(agent) - return agent - - def create_callable_chain(self, chain_definition: List[CallableChainItem]): - chain = {"definition": [item._asdict() for item in chain_definition]} - self.callable_chains.append(chain) - return chain - - def register_function(self, function_definition: FunctionDefinition): - if function_definition.identifier in self.functions: - raise ValueError(f"Function {function_definition.identifier} is already registered.") - - self.functions[function_definition.identifier] = function_definition - - def export_configuration(self, format_type: str = 'json'): - # Now exporting both swarms and callable chains - config = {"swarms": self.swarms, "callable_chains": self.callable_chains} - if format_type == "json": - return json.dumps(config) - elif format_type == "pickle": - return pickle.dumps(config) - - def load_configuration(self, config_data, format_type: str = 'json'): - # Loading both swarms and callable chains - config = json.loads(config_data) if format_type == "json" else pickle.loads(config_data) - self.swarms = config.get("swarms", []) - self.callable_chains = config.get("callable_chains", []) \ No newline at end of file diff --git a/pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/__init__.py b/pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/__init__.py deleted file mode 100644 index 61f84eae6..000000000 --- a/pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from swarmauri.utils._lazy_import import _lazy_import - -# List of swarms names (file names without the ".py" extension) and corresponding class names -swarms_files = [("swarmauri.swarms.concrete.SimpleSwarmFactory", "SimpleSwarmFactory")] - -# Lazy loading of swarms classes, storing them in variables -for module_name, class_name in swarms_files: - globals()[class_name] = _lazy_import(module_name, class_name) - -# Adding the lazy-loaded swarms classes to __all__ -__all__ = [class_name for _, class_name in swarms_files] diff --git a/pkgs/swarmauri/swarmauri/swarms/base/SwarmBase.py b/pkgs/swarmauri/swarmauri/swarms/base/SwarmBase.py index 45972f930..88ba5e968 100644 --- a/pkgs/swarmauri/swarmauri/swarms/base/SwarmBase.py +++ b/pkgs/swarmauri/swarmauri/swarms/base/SwarmBase.py @@ -50,7 +50,6 @@ def agents(self) -> List[Any]: def queue_size(self) -> int: return self._task_queue.qsize() - def get_swarm_status(self) -> Dict[int, SwarmStatus]: return self._status @@ -60,7 +59,8 @@ async def _process_task(self, agent_id: int, task: Any, **kwargs) -> Any: for _ in range(self.max_retries): try: result = await asyncio.wait_for( - self._execute_task(task, agent_id, **kwargs), timeout=self.agent_timeout + self._execute_task(task, agent_id, **kwargs), + timeout=self.agent_timeout, ) self._status[agent_id] = SwarmStatus.COMPLETED return result From f38199f81480b3a18456a81a036aaa5cdcf9e92e Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 07:51:19 -0600 Subject: [PATCH 088/168] Update TransportBase.py --- pkgs/swarmauri/swarmauri/transport/base/TransportBase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py b/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py index a25cff190..ddb630e26 100644 --- a/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py +++ b/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py @@ -5,7 +5,7 @@ from swarmauri_core.transport.ITransport import ITransport -class TransportationProtocol(Enum): +class TransportProtocol(Enum): """ Enumeration of transportation protocols supported by the transport layer """ @@ -17,7 +17,7 @@ class TransportationProtocol(Enum): class TransportBase(ITransport, ComponentBase): - allowed_protocols: List[TransportationProtocol] = [] + allowed_protocols: List[TransportProtocol] = [] resource: Optional[str] = Field(default=ResourceTypes.TRANSPORT.value, frozen=True) model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) type: Literal["TransportBase"] = "TransportBase" From c6996722652382ec7a391a5b5dd31bf560441ff2 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 08:01:41 -0600 Subject: [PATCH 089/168] Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index fbe485428..5846c5c09 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -123,7 +123,13 @@ jobs: curl -sSL https://install.python-poetry.org | python3 - echo "PATH=$HOME/.local/bin:$PATH" >> $GITHUB_ENV + - name: Install swarmauri_core package dependency + run: | + cd pkgs/core + poetry install --no-cache --all-extras -vv + - name: Install package dependencies + if: matrix.package_tests.package != 'core' run: | cd pkgs/${{ matrix.package_tests.package }} poetry install --no-cache --all-extras -vv From d27a0b754d849f8c49eaec7a9925f82cfbb08621 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 08:03:12 -0600 Subject: [PATCH 090/168] core - Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index 2270780ce..e913a70a3 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -15,17 +15,25 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.13" + +# Swarmauri swarmauri_core = { path = "./pkgs/core/swarmauri_core" } + +# Dependencies toml = "^0.10.2" httpx = "^0.27.0" joblib = "^1.4.0" numpy = "*" pandas = "*" pydantic = "^2.9.2" -Pillow = ">=8.0,<11.0" typing_extensions = "*" + +# We should remove and only rely on httpx requests = "*" +# This should be set to optional also +Pillow = ">=8.0,<11.0" + # Optional dependencies with versions specified aiofiles = { version = "24.1.0", optional = true } aiohttp = { version = "^3.10.10", optional = true } From 33c0d38cd059f1dc29eb532966190360f0a5b4a4 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 08:04:19 -0600 Subject: [PATCH 091/168] swarm - Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index e913a70a3..e67b513e6 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -14,6 +14,7 @@ classifiers = [ ] [tool.poetry.dependencies] +# Python Version python = ">=3.10,<3.13" # Swarmauri From b52ea84ad73d2d9fc5e1713b0b14de293046a29b Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 08:08:26 -0600 Subject: [PATCH 092/168] Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index e67b513e6..2faf92b76 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -18,7 +18,7 @@ classifiers = [ python = ">=3.10,<3.13" # Swarmauri -swarmauri_core = { path = "./pkgs/core/swarmauri_core" } +swarmauri_core = { path = "./pkgs/core" } # Dependencies toml = "^0.10.2" From 040796268bc8c46f43fe008ef164a5a7dea6f21b Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 08:09:48 -0600 Subject: [PATCH 093/168] Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index 2faf92b76..77d3b1211 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -18,7 +18,7 @@ classifiers = [ python = ">=3.10,<3.13" # Swarmauri -swarmauri_core = { path = "./pkgs/core" } +swarmauri_core = { path = "/pkgs/core" } # Dependencies toml = "^0.10.2" From de08f0b57ce06fe74fb5a084f78a71bdee58a6e3 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 08:13:03 -0600 Subject: [PATCH 094/168] swarm - Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index 77d3b1211..140194615 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -18,7 +18,7 @@ classifiers = [ python = ">=3.10,<3.13" # Swarmauri -swarmauri_core = { path = "/pkgs/core" } +swarmauri_core = { path = "../pkgs/core" } # Dependencies toml = "^0.10.2" From b1b7a5fd573ab8444e262b5a615618584923531d Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 08:15:49 -0600 Subject: [PATCH 095/168] Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index 140194615..dd86ad619 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -18,7 +18,7 @@ classifiers = [ python = ">=3.10,<3.13" # Swarmauri -swarmauri_core = { path = "../pkgs/core" } +swarmauri_core = { path = "../core" } # Dependencies toml = "^0.10.2" From 427e5112aec2ca68f36a86c83f677ec40dde6f04 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:23:38 -0600 Subject: [PATCH 096/168] swarm - Update PubSubTransport.py --- .../swarmauri/swarmauri/transport/concrete/PubSubTransport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py b/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py index 1bf1ea055..f44654791 100644 --- a/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py +++ b/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py @@ -1,11 +1,11 @@ from typing import Dict, Any, List, Optional, Set, Literal from uuid import uuid4 import asyncio -from swarmauri.transport.base.TransportBase import TransportBase, TransportationProtocol +from swarmauri.transport.base.TransportBase import TransportBase, TransportProtocol class PubSubTransport(TransportBase): - allowed_protocols: List[TransportationProtocol] = [TransportationProtocol.PUBSUB] + allowed_protocols: List[TransportProtocol] = [TransportProtocol.PUBSUB] _topics: Dict[str, Set[str]] = {} # Topic to subscriber mappings _subscribers: Dict[str, asyncio.Queue] = {} type: Literal["PubSubTransport"] = "PubSubTransport" From 190fef42c2a6fb88d58ba186f652855656771479 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:27:10 -0600 Subject: [PATCH 097/168] swarm - transport => transports --- .../core/swarmauri_core/{transport => transports}/ITransport.py | 0 pkgs/core/swarmauri_core/{transport => transports}/__init__.py | 0 pkgs/swarmauri/swarmauri/{transport => transports}/__init__.py | 0 .../swarmauri/{transport => transports}/base/TransportBase.py | 2 +- .../swarmauri/{transport => transports}/base/__init__.py | 0 .../{transport => transports}/concrete/PubSubTransport.py | 2 +- .../swarmauri/{transport => transports}/concrete/__init__.py | 0 .../unit/{transport => transports}/PubSubTransport_unit_test.py | 2 +- 8 files changed, 3 insertions(+), 3 deletions(-) rename pkgs/core/swarmauri_core/{transport => transports}/ITransport.py (100%) rename pkgs/core/swarmauri_core/{transport => transports}/__init__.py (100%) rename pkgs/swarmauri/swarmauri/{transport => transports}/__init__.py (100%) rename pkgs/swarmauri/swarmauri/{transport => transports}/base/TransportBase.py (96%) rename pkgs/swarmauri/swarmauri/{transport => transports}/base/__init__.py (100%) rename pkgs/swarmauri/swarmauri/{transport => transports}/concrete/PubSubTransport.py (97%) rename pkgs/swarmauri/swarmauri/{transport => transports}/concrete/__init__.py (100%) rename pkgs/swarmauri/tests/unit/{transport => transports}/PubSubTransport_unit_test.py (98%) diff --git a/pkgs/core/swarmauri_core/transport/ITransport.py b/pkgs/core/swarmauri_core/transports/ITransport.py similarity index 100% rename from pkgs/core/swarmauri_core/transport/ITransport.py rename to pkgs/core/swarmauri_core/transports/ITransport.py diff --git a/pkgs/core/swarmauri_core/transport/__init__.py b/pkgs/core/swarmauri_core/transports/__init__.py similarity index 100% rename from pkgs/core/swarmauri_core/transport/__init__.py rename to pkgs/core/swarmauri_core/transports/__init__.py diff --git a/pkgs/swarmauri/swarmauri/transport/__init__.py b/pkgs/swarmauri/swarmauri/transports/__init__.py similarity index 100% rename from pkgs/swarmauri/swarmauri/transport/__init__.py rename to pkgs/swarmauri/swarmauri/transports/__init__.py diff --git a/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py b/pkgs/swarmauri/swarmauri/transports/base/TransportBase.py similarity index 96% rename from pkgs/swarmauri/swarmauri/transport/base/TransportBase.py rename to pkgs/swarmauri/swarmauri/transports/base/TransportBase.py index ddb630e26..370ed1c8a 100644 --- a/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py +++ b/pkgs/swarmauri/swarmauri/transports/base/TransportBase.py @@ -2,7 +2,7 @@ from pydantic import ConfigDict, Field from enum import Enum, auto from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes -from swarmauri_core.transport.ITransport import ITransport +from swarmauri_core.transports.ITransport import ITransport class TransportProtocol(Enum): diff --git a/pkgs/swarmauri/swarmauri/transport/base/__init__.py b/pkgs/swarmauri/swarmauri/transports/base/__init__.py similarity index 100% rename from pkgs/swarmauri/swarmauri/transport/base/__init__.py rename to pkgs/swarmauri/swarmauri/transports/base/__init__.py diff --git a/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py b/pkgs/swarmauri/swarmauri/transports/concrete/PubSubTransport.py similarity index 97% rename from pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py rename to pkgs/swarmauri/swarmauri/transports/concrete/PubSubTransport.py index f44654791..48303ad6a 100644 --- a/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py +++ b/pkgs/swarmauri/swarmauri/transports/concrete/PubSubTransport.py @@ -1,7 +1,7 @@ from typing import Dict, Any, List, Optional, Set, Literal from uuid import uuid4 import asyncio -from swarmauri.transport.base.TransportBase import TransportBase, TransportProtocol +from swarmauri.transports.base.TransportBase import TransportBase, TransportProtocol class PubSubTransport(TransportBase): diff --git a/pkgs/swarmauri/swarmauri/transport/concrete/__init__.py b/pkgs/swarmauri/swarmauri/transports/concrete/__init__.py similarity index 100% rename from pkgs/swarmauri/swarmauri/transport/concrete/__init__.py rename to pkgs/swarmauri/swarmauri/transports/concrete/__init__.py diff --git a/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py b/pkgs/swarmauri/tests/unit/transports/PubSubTransport_unit_test.py similarity index 98% rename from pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py rename to pkgs/swarmauri/tests/unit/transports/PubSubTransport_unit_test.py index c8918b086..fb5aeeb5e 100644 --- a/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py +++ b/pkgs/swarmauri/tests/unit/transports/PubSubTransport_unit_test.py @@ -2,7 +2,7 @@ import asyncio from uuid import UUID from typing import Any -from swarmauri.transport.concrete.PubSubTransport import ( +from swarmauri.transports.concrete.PubSubTransport import ( PubSubTransport, ) from swarmauri.utils.timeout_wrapper import timeout From 00885b9654859ea8ebc0b1179b386fd1c9120c31 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:29:05 -0600 Subject: [PATCH 098/168] swarm - update transports/concrete/__init__.py --- pkgs/swarmauri/swarmauri/transports/concrete/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/swarmauri/transports/concrete/__init__.py b/pkgs/swarmauri/swarmauri/transports/concrete/__init__.py index 893b547b4..b9f008bb5 100644 --- a/pkgs/swarmauri/swarmauri/transports/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/transports/concrete/__init__.py @@ -2,7 +2,7 @@ # List of transport names (file names without the ".py" extension) and corresponding class names transport_files = [ - ("swarmauri.transport.concrete.PubSubTransport", "PubSubTransport"), + ("swarmauri.transports.concrete.PubSubTransport", "PubSubTransport"), ] # Lazy loading of transport classes, storing them in variables From cb97369bf773b3da2f19d99a8ca517b2fd3b311f Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:31:29 -0600 Subject: [PATCH 099/168] Update __init__.py --- pkgs/swarmauri/swarmauri/transports/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/swarmauri/transports/__init__.py b/pkgs/swarmauri/swarmauri/transports/__init__.py index f3ada0e57..4de373c3c 100644 --- a/pkgs/swarmauri/swarmauri/transports/__init__.py +++ b/pkgs/swarmauri/swarmauri/transports/__init__.py @@ -1 +1 @@ -from swarmauri.transport.concrete import * +from swarmauri.transports.concrete import * From 241b294a5e201a22bfb665cd853d30f81d823f8b Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:39:23 -0600 Subject: [PATCH 100/168] swarm - ControlPanelBase => ControlPanel --- .../tests/unit/control_panels/ControlPanel_unit_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py index 8ce020ae0..be3fa2389 100644 --- a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py +++ b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py @@ -1,6 +1,6 @@ import pytest from unittest.mock import MagicMock -from swarmauri.control_panels.base.ControlPanelBase import ControlPanelBase +from swarmauri.control_panels.concrete.ControlPanel import ControlPanel from swarmauri.factories.base.FactoryBase import FactoryBase from swarmauri.service_registries.base.ServiceRegistryBase import ServiceRegistryBase from swarmauri.task_mgt_strategies.base.TaskMgtStrategyBase import TaskMgtStrategyBase @@ -9,13 +9,13 @@ @pytest.fixture def control_panel(): - """Fixture to create a ControlPanelBase instance with mocked dependencies.""" + """Fixture to create a ControlPanel instance with mocked dependencies.""" agent_factory = MagicMock(spec=FactoryBase) service_registry = MagicMock(spec=ServiceRegistryBase) task_mgt_strategy = MagicMock(spec=TaskMgtStrategyBase) transport = MagicMock(spec=TransportBase) - return ControlPanelBase( + return ControlPanel( agent_factory=agent_factory, service_registry=service_registry, task_mgt_strategy=task_mgt_strategy, From 1b40288a76e53e0dbcc3f71f2eb75c7805d8d54c Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:44:55 -0600 Subject: [PATCH 101/168] swarm - Update ControlPanelBase.py --- .../swarmauri/control_panels/base/ControlPanelBase.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py b/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py index 2c7d443a7..2e36f48cc 100644 --- a/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py +++ b/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py @@ -1,8 +1,7 @@ -from swarmauri_core import ComponentBase +from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes from swarmauri_core.control_panels.IControlPanel import IControlPlane from typing import Any, List, Literal from pydantic import Field, ConfigDict -from swarmauri_core.ComponentBase import ResourceTypes from swarmauri.service_registries.base.ServiceRegistryBase import ServiceRegistryBase from swarmauri.factories.base.FactoryBase import FactoryBase from swarmauri.task_mgt_strategies.base.TaskMgtStrategyBase import TaskMgtStrategyBase From c0ad132d2355f1d4bfb90504a7122dab324c1c13 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:48:37 -0600 Subject: [PATCH 102/168] swarm - test serialization of magic mock --- .../control_panels/ControlPanel_unit_test.py | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py index be3fa2389..fa77f283e 100644 --- a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py +++ b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py @@ -6,15 +6,28 @@ from swarmauri.task_mgt_strategies.base.TaskMgtStrategyBase import TaskMgtStrategyBase from swarmauri.transports.base.TransportBase import TransportBase +from unittest.mock import MagicMock +from pydantic import BaseModel + +class SerializableMagicMock(MagicMock, BaseModel): + """A MagicMock class that can be serialized using Pydantic.""" + + def dict(self, *args, **kwargs): + """Serialize the mock object to a dictionary.""" + return {"mock_name": self._mock_name, "calls": self.mock_calls} + + def json(self, *args, **kwargs): + """Serialize the mock object to a JSON string.""" + return super().json(*args, **kwargs) @pytest.fixture def control_panel(): """Fixture to create a ControlPanel instance with mocked dependencies.""" - agent_factory = MagicMock(spec=FactoryBase) - service_registry = MagicMock(spec=ServiceRegistryBase) - task_mgt_strategy = MagicMock(spec=TaskMgtStrategyBase) - transport = MagicMock(spec=TransportBase) - + agent_factory = SerializableMagicMock(spec=FactoryBase) + service_registry = SerializableMagicMock(spec=ServiceRegistryBase) + task_mgt_strategy = SerializableMagicMock(spec=TaskMgtStrategyBase) + transport = SerializableMagicMock(spec=TransportBase) + return ControlPanel( agent_factory=agent_factory, service_registry=service_registry, From 52089295fab5e5f6b806c6e202a9598e711da11f Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:53:22 -0600 Subject: [PATCH 103/168] swarm - test mock --- .../control_panels/ControlPanel_unit_test.py | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py index fa77f283e..93065439d 100644 --- a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py +++ b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py @@ -22,12 +22,28 @@ def json(self, *args, **kwargs): @pytest.fixture def control_panel(): - """Fixture to create a ControlPanel instance with mocked dependencies.""" + """Fixture to create a fully mocked ControlPanel instance with serializable mocks.""" + + # Create serializable mocks for all dependencies agent_factory = SerializableMagicMock(spec=FactoryBase) + agent_factory.create_agent = SerializableMagicMock(return_value="MockAgent") + agent_factory.get_agent_by_name = SerializableMagicMock(return_value="MockAgent") + agent_factory.delete_agent = SerializableMagicMock() + agent_factory.get_agents = SerializableMagicMock(return_value=["MockAgent1", "MockAgent2"]) + service_registry = SerializableMagicMock(spec=ServiceRegistryBase) + service_registry.register_service = SerializableMagicMock() + service_registry.unregister_service = SerializableMagicMock() + service_registry.get_services = SerializableMagicMock(return_value=["service1", "service2"]) + task_mgt_strategy = SerializableMagicMock(spec=TaskMgtStrategyBase) + task_mgt_strategy.add_task = SerializableMagicMock() + task_mgt_strategy.process_tasks = SerializableMagicMock() + task_mgt_strategy.assign_task = SerializableMagicMock() + transport = SerializableMagicMock(spec=TransportBase) - + + # Return the ControlPanel instance with mocked dependencies return ControlPanel( agent_factory=agent_factory, service_registry=service_registry, @@ -35,7 +51,6 @@ def control_panel(): transport=transport, ) - def test_create_agent(control_panel): """Test the create_agent method.""" agent_name = "agent1" From 4bde463231a590c89b33e4dffeff8b6abfea890e Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:54:51 -0600 Subject: [PATCH 104/168] Update ControlPanel_unit_test.py --- .../tests/unit/control_panels/ControlPanel_unit_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py index 93065439d..78d266d46 100644 --- a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py +++ b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py @@ -70,7 +70,7 @@ def test_create_agent(control_panel): control_panel.service_registry.register_service.assert_called_once_with( agent_name, {"role": agent_role, "status": "active"} ) - assert result == agent + assert result == "MockAgent" def test_remove_agent(control_panel): From 8b8c77826535c2f11d8e7e4a0b4a45e53cdbd0f9 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:58:53 -0600 Subject: [PATCH 105/168] swarm - fix pipeline inheritance --- pkgs/swarmauri/swarmauri/pipelines/base/PipelineBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/swarmauri/pipelines/base/PipelineBase.py b/pkgs/swarmauri/swarmauri/pipelines/base/PipelineBase.py index da91427b9..6cc302c2b 100644 --- a/pkgs/swarmauri/swarmauri/pipelines/base/PipelineBase.py +++ b/pkgs/swarmauri/swarmauri/pipelines/base/PipelineBase.py @@ -5,7 +5,7 @@ import uuid -class PipelineBase(ComponentBase, IPipeline): +class PipelineBase(IPipeline, ComponentBase): """ Base class providing default behavior for task orchestration, error handling, and result aggregation. From 221666a9271754af7cb69edd0d2c6909a2867038 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:12:55 -0600 Subject: [PATCH 106/168] Create manage_issues.py --- scripts/manage_issues.py | 90 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 scripts/manage_issues.py diff --git a/scripts/manage_issues.py b/scripts/manage_issues.py new file mode 100644 index 000000000..884b8a33f --- /dev/null +++ b/scripts/manage_issues.py @@ -0,0 +1,90 @@ +import os +import json +import requests + +# GitHub API settings +GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") +REPO = os.getenv("REPO") +HEADERS = { + "Authorization": f"Bearer {GITHUB_TOKEN}", + "Accept": "application/vnd.github.v3+json", +} + +ISSUE_LABEL = "pytest-failure" + + +def load_pytest_results(report_file="report.json"): + """Load pytest results from the JSON report.""" + if not os.path.exists(report_file): + print("No pytest report file found.") + return [] + + with open(report_file, "r") as f: + report = json.load(f) + + # Extract failed test cases + failures = [] + for test in report.get("tests", []): + if test["outcome"] == "failed": + failures.append({ + "name": test["name"], + "path": test["nodeid"], + "message": test.get("call", {}).get("longrepr", "No details provided") + }) + return failures + + +def get_existing_issues(): + """Retrieve all existing issues with the pytest-failure label.""" + url = f"https://api.github.com/repos/{REPO}/issues" + params = {"labels": ISSUE_LABEL, "state": "open"} + response = requests.get(url, headers=HEADERS, params=params) + response.raise_for_status() + return response.json() + + +def create_issue(test): + """Create a new GitHub issue for the test failure.""" + url = f"https://api.github.com/repos/{REPO}/issues" + data = { + "title": f"Test Failure: {test['name']}", + "body": f"### Test Case\n```\n{test['path']}\n```\n### Failure Details\n{test['message']}", + "labels": [ISSUE_LABEL] + } + response = requests.post(url, headers=HEADERS, json=data) + response.raise_for_status() + print(f"Issue created for {test['name']}") + + +def add_comment_to_issue(issue_number, test): + """Add a comment to an existing GitHub issue.""" + url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" + data = {"body": f"New failure detected:\n```\n{test['path']}\n```\n### Details\n{test['message']}"} + response = requests.post(url, headers=HEADERS, json=data) + response.raise_for_status() + print(f"Comment added to issue {issue_number} for {test['name']}") + + +def process_failures(): + """Process pytest failures and manage GitHub issues.""" + failures = load_pytest_results() + if not failures: + print("No test failures found.") + return + + existing_issues = get_existing_issues() + + for test in failures: + issue_exists = False + for issue in existing_issues: + if test["name"] in issue["title"]: + add_comment_to_issue(issue["number"], test) + issue_exists = True + break + + if not issue_exists: + create_issue(test) + + +if __name__ == "__main__": + process_failures() From b638af5ecec34d05bb67d5062c4d7b429c6b147a Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:14:28 -0600 Subject: [PATCH 107/168] Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index 5846c5c09..d1369e6a6 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -134,9 +134,18 @@ jobs: cd pkgs/${{ matrix.package_tests.package }} poetry install --no-cache --all-extras -vv - - name: Run all tests for the package + - name: Run tests and save results run: | echo "Running tests for package: ${{ matrix.package_tests.package }}" echo "Test files: ${{ matrix.package_tests.tests }}" cd pkgs/${{ matrix.package_tests.package }} - poetry run pytest ${{ matrix.package_tests.tests }} + poetry run pytest ${{ matrix.package_tests.tests }} --tb=short --json-report --json-report-file=pytest_results.json || true + + - name: Process test results and manage issues + if: always() + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + run: | + python .github/scripts/manage_issues.py + From ee76f4d4e5aeb93dc0cef9e198748c3e3a30607c Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:16:40 -0600 Subject: [PATCH 108/168] Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index d1369e6a6..2d0dbf124 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -147,5 +147,5 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} run: | - python .github/scripts/manage_issues.py + python ../.github/scripts/manage_issues.py From bcb97c5c5b31cb404915eee5884cbfbedf04a784 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:19:59 -0600 Subject: [PATCH 109/168] Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index 2d0dbf124..533e9f79b 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -122,6 +122,7 @@ jobs: run: | curl -sSL https://install.python-poetry.org | python3 - echo "PATH=$HOME/.local/bin:$PATH" >> $GITHUB_ENV + poetry run pip install pytest-json-report - name: Install swarmauri_core package dependency run: | From d7b0f245c3e3c5af3e2477828f6ab70d553e4ec4 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:21:04 -0600 Subject: [PATCH 110/168] swarm - Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index dd86ad619..13924f3de 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -95,10 +95,12 @@ pytest = "^8.0" pytest-asyncio = ">=0.24.0" pytest-timeout = "^2.3.1" pytest-xdist = "^3.6.1" +pytest-json-report = "^1.5.0" python-dotenv = "^1.0.0" jsonschema = "^4.18.5" ipython = "8.28.0" + [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" From be5399a59a02fcae984a0e759ed78cb4d996990d Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:21:19 -0600 Subject: [PATCH 111/168] Update pyproject.toml --- pkgs/core/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/core/pyproject.toml b/pkgs/core/pyproject.toml index 887619499..5558e0cd5 100644 --- a/pkgs/core/pyproject.toml +++ b/pkgs/core/pyproject.toml @@ -25,6 +25,7 @@ flake8 = "^7.0" # Add flake8 as a development dependency pytest = "^8.0" # Ensure pytest is also added if you run tests pytest-asyncio = ">=0.24.0" pytest-xdist = "^3.6.1" +pytest-json-report = "^1.5.0" [build-system] requires = ["poetry-core>=1.0.0"] From a34f8346a360a7d3c1df3143b03f06ba47cf67fc Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:21:37 -0600 Subject: [PATCH 112/168] Update pyproject.toml --- pkgs/community/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/community/pyproject.toml b/pkgs/community/pyproject.toml index db711b327..d8feb1490 100644 --- a/pkgs/community/pyproject.toml +++ b/pkgs/community/pyproject.toml @@ -76,6 +76,7 @@ flake8 = "^7.0" pytest = "^8.0" pytest-asyncio = ">=0.24.0" pytest-xdist = "^3.6.1" +pytest-json-report = "^1.5.0" python-dotenv = "*" [build-system] From 51a86d159ba8a2a18e6b462153071dd005001d97 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:22:17 -0600 Subject: [PATCH 113/168] exp - Update pyproject.toml --- pkgs/experimental/pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/experimental/pyproject.toml b/pkgs/experimental/pyproject.toml index 2bcdb3b7f..a9bef7a00 100644 --- a/pkgs/experimental/pyproject.toml +++ b/pkgs/experimental/pyproject.toml @@ -31,6 +31,8 @@ typing_extensions = "*" flake8 = "^7.0" # Add flake8 as a development dependency pytest = "^8.0" # Ensure pytest is also added if you run tests pytest-asyncio = ">=0.24.0" +pytest-xdist = "^3.6.1" +pytest-json-report = "^1.5.0" [build-system] requires = ["poetry-core>=1.0.0"] From 2a37ef5c222b997df13325ca0badc8b6396437dd Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:23:59 -0600 Subject: [PATCH 114/168] Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index 533e9f79b..2d0dbf124 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -122,7 +122,6 @@ jobs: run: | curl -sSL https://install.python-poetry.org | python3 - echo "PATH=$HOME/.local/bin:$PATH" >> $GITHUB_ENV - poetry run pip install pytest-json-report - name: Install swarmauri_core package dependency run: | From 49be12f05cfb6c32626e98113c538bd5c9770f7d Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:25:35 -0600 Subject: [PATCH 115/168] cicd - Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index 2d0dbf124..23332133d 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -147,5 +147,5 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} run: | - python ../.github/scripts/manage_issues.py + python scripts/manage_issues.py From 4a1d219205d41d401fc4ddc1cd676c458fae9355 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:27:50 -0600 Subject: [PATCH 116/168] swarm - Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index 13924f3de..6177ffd54 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -99,7 +99,7 @@ pytest-json-report = "^1.5.0" python-dotenv = "^1.0.0" jsonschema = "^4.18.5" ipython = "8.28.0" - +requests = "*" [build-system] requires = ["poetry-core>=1.0.0"] From 1d1489d9eb61ada91f39fae4a96ff452f127c834 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:29:07 -0600 Subject: [PATCH 117/168] swarm - Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index 6177ffd54..624a74b0e 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -30,7 +30,7 @@ pydantic = "^2.9.2" typing_extensions = "*" # We should remove and only rely on httpx -requests = "*" +requests = "^2.32.3" # This should be set to optional also Pillow = ">=8.0,<11.0" @@ -98,8 +98,8 @@ pytest-xdist = "^3.6.1" pytest-json-report = "^1.5.0" python-dotenv = "^1.0.0" jsonschema = "^4.18.5" -ipython = "8.28.0" -requests = "*" +ipython = "^8.28.0" +requests = "^2.32.3" [build-system] requires = ["poetry-core>=1.0.0"] From 15cb5276162d28f98ed5cd0fafb23366999651d8 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:33:51 -0600 Subject: [PATCH 118/168] Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index 23332133d..612a77ee4 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -147,5 +147,5 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} run: | - python scripts/manage_issues.py + poetry run python .github/scripts/manage_issues.py From b2a9a6f1303db05e07a6cef4bacbb00fd13dd193 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:36:27 -0600 Subject: [PATCH 119/168] cicd - Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index 612a77ee4..384ee2cd1 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -147,5 +147,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} run: | - poetry run python .github/scripts/manage_issues.py + cd pkgs/${{ matrix.package_tests.package }} + poetry run python scripts/manage_issues.py From 7208542eec28750a6cd7cc689fbf350ead71e31f Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:39:59 -0600 Subject: [PATCH 120/168] cicd - Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index 384ee2cd1..07c1cdf4c 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -147,6 +147,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} run: | - cd pkgs/${{ matrix.package_tests.package }} - poetry run python scripts/manage_issues.py + cd pkgs/swarmauri # Change to the directory containing pyproject.toml + poetry run python ../../scripts/manage_issues.py + From 74317ce83dceac3075ee78b5af17a6ba850516eb Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:42:50 -0600 Subject: [PATCH 121/168] cicd - Update manage_issues.py --- scripts/manage_issues.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/scripts/manage_issues.py b/scripts/manage_issues.py index 884b8a33f..df844009d 100644 --- a/scripts/manage_issues.py +++ b/scripts/manage_issues.py @@ -1,6 +1,7 @@ import os import json import requests +import argparse # GitHub API settings GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") @@ -12,11 +13,21 @@ ISSUE_LABEL = "pytest-failure" - -def load_pytest_results(report_file="report.json"): +def parse_arguments(): + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description="Manage GitHub Issues for Test Failures") + parser.add_argument( + "--results-file", + type=str, + default="pytest_results.json", + help="Path to the pytest JSON report file (default: pytest_results.json)", + ) + return parser.parse_args() + +def load_pytest_results(report_file): """Load pytest results from the JSON report.""" if not os.path.exists(report_file): - print("No pytest report file found.") + print(f"Report file not found: {report_file}") return [] with open(report_file, "r") as f: @@ -33,7 +44,6 @@ def load_pytest_results(report_file="report.json"): }) return failures - def get_existing_issues(): """Retrieve all existing issues with the pytest-failure label.""" url = f"https://api.github.com/repos/{REPO}/issues" @@ -42,7 +52,6 @@ def get_existing_issues(): response.raise_for_status() return response.json() - def create_issue(test): """Create a new GitHub issue for the test failure.""" url = f"https://api.github.com/repos/{REPO}/issues" @@ -55,7 +64,6 @@ def create_issue(test): response.raise_for_status() print(f"Issue created for {test['name']}") - def add_comment_to_issue(issue_number, test): """Add a comment to an existing GitHub issue.""" url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" @@ -64,10 +72,9 @@ def add_comment_to_issue(issue_number, test): response.raise_for_status() print(f"Comment added to issue {issue_number} for {test['name']}") - -def process_failures(): +def process_failures(report_file): """Process pytest failures and manage GitHub issues.""" - failures = load_pytest_results() + failures = load_pytest_results(report_file) if not failures: print("No test failures found.") return @@ -85,6 +92,6 @@ def process_failures(): if not issue_exists: create_issue(test) - if __name__ == "__main__": - process_failures() + args = parse_arguments() + process_failures(args.results_file) From 5732a54f60db13776dd43247114cb8f36aaa0424 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:44:40 -0600 Subject: [PATCH 122/168] Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index 07c1cdf4c..57cfafa92 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -148,6 +148,6 @@ jobs: REPO: ${{ github.repository }} run: | cd pkgs/swarmauri # Change to the directory containing pyproject.toml - poetry run python ../../scripts/manage_issues.py + poetry run python ../../scripts/manage_issues.py --results-file=pytest_results.json From 89a047b5dbb7a49c793361249b41f650cef3f583 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:46:34 -0600 Subject: [PATCH 123/168] cicd - Update manage_issues.py --- scripts/manage_issues.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/manage_issues.py b/scripts/manage_issues.py index df844009d..8ad253e6a 100644 --- a/scripts/manage_issues.py +++ b/scripts/manage_issues.py @@ -36,14 +36,20 @@ def load_pytest_results(report_file): # Extract failed test cases failures = [] for test in report.get("tests", []): - if test["outcome"] == "failed": + if test.get("outcome") == "failed": + # Safely extract keys with defaults + test_name = test.get("nodeid", "Unknown test") + test_path = test.get("nodeid", "Unknown path") + failure_message = test.get("call", {}).get("longrepr", "No failure details available") + failures.append({ - "name": test["name"], - "path": test["nodeid"], - "message": test.get("call", {}).get("longrepr", "No details provided") + "name": test_name, + "path": test_path, + "message": failure_message }) return failures + def get_existing_issues(): """Retrieve all existing issues with the pytest-failure label.""" url = f"https://api.github.com/repos/{REPO}/issues" From c5ad24ffeed5cb39ea874cbbb2259521776fa0d4 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:48:52 -0600 Subject: [PATCH 124/168] cicd - Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index 57cfafa92..c593a21a0 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -99,6 +99,9 @@ jobs: run-tests: needs: detect-changed-files runs-on: ubuntu-latest + permissions: + issues: write + contents: read if: ${{ needs.detect-changed-files.outputs.matrix != '[]' }} strategy: fail-fast: false From 684054d60f3213b6a0a762c791241a248f0ba8e8 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:55:26 -0600 Subject: [PATCH 125/168] cicd - Update manage_issues.py --- scripts/manage_issues.py | 80 ++++++++++++++++++++++++++++++---------- 1 file changed, 60 insertions(+), 20 deletions(-) diff --git a/scripts/manage_issues.py b/scripts/manage_issues.py index 8ad253e6a..b7da50ab2 100644 --- a/scripts/manage_issues.py +++ b/scripts/manage_issues.py @@ -11,7 +11,9 @@ "Accept": "application/vnd.github.v3+json", } -ISSUE_LABEL = "pytest-failure" +BASE_BRANCH = os.getenv("GITHUB_REF", "unknown").split("/")[-1] +COMMIT_SHA = os.getenv("GITHUB_SHA", "unknown") +WORKFLOW_RUN_URL = os.getenv("GITHUB_SERVER_URL", "https://github.com") + f"/{REPO}/actions/runs/{os.getenv('GITHUB_RUN_ID', 'unknown')}" def parse_arguments(): """Parse command-line arguments.""" @@ -22,6 +24,12 @@ def parse_arguments(): default="pytest_results.json", help="Path to the pytest JSON report file (default: pytest_results.json)", ) + parser.add_argument( + "--package", + type=str, + required=True, + help="Name of the matrix package where the error occurred.", + ) return parser.parse_args() def load_pytest_results(report_file): @@ -33,52 +41,84 @@ def load_pytest_results(report_file): with open(report_file, "r") as f: report = json.load(f) - # Extract failed test cases failures = [] for test in report.get("tests", []): if test.get("outcome") == "failed": - # Safely extract keys with defaults test_name = test.get("nodeid", "Unknown test") - test_path = test.get("nodeid", "Unknown path") failure_message = test.get("call", {}).get("longrepr", "No failure details available") - failures.append({ "name": test_name, - "path": test_path, + "path": test.get("nodeid", "Unknown path"), "message": failure_message }) return failures - def get_existing_issues(): """Retrieve all existing issues with the pytest-failure label.""" url = f"https://api.github.com/repos/{REPO}/issues" - params = {"labels": ISSUE_LABEL, "state": "open"} + params = {"labels": "pytest-failure", "state": "open"} response = requests.get(url, headers=HEADERS, params=params) response.raise_for_status() return response.json() -def create_issue(test): +def create_issue(test, package): """Create a new GitHub issue for the test failure.""" url = f"https://api.github.com/repos/{REPO}/issues" + + # Construct the issue body with enhanced documentation data = { - "title": f"Test Failure: {test['name']}", - "body": f"### Test Case\n```\n{test['path']}\n```\n### Failure Details\n{test['message']}", - "labels": [ISSUE_LABEL] + "title": f"[Test Case Failure]: {test['name']}", + "body": f""" +### Test Case: +{test['path']} + +### Failure Details: +{test['message']} + +--- + +### Context: +- **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) +- **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) +- **Workflow Run**: [View Run]({WORKFLOW_RUN_URL}) +- **Matrix Package**: `{package}` + +--- + +### Label: +This issue is auto-labeled for the `{package}` package. +""", + "labels": ["pytest-failure", package] } response = requests.post(url, headers=HEADERS, json=data) response.raise_for_status() - print(f"Issue created for {test['name']}") + print(f"Issue created for {test['name']} in package '{package}'.") -def add_comment_to_issue(issue_number, test): +def add_comment_to_issue(issue_number, test, package): """Add a comment to an existing GitHub issue.""" url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" - data = {"body": f"New failure detected:\n```\n{test['path']}\n```\n### Details\n{test['message']}"} + data = {"body": f""" +New failure detected: + +### Test Case: +{test['path']} + +### Details: +{test['message']} + +--- + +### Context: +- **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) +- **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) +- **Workflow Run**: [View Run]({WORKFLOW_RUN_URL}) +- **Matrix Package**: `{package}` +"""} response = requests.post(url, headers=HEADERS, json=data) response.raise_for_status() - print(f"Comment added to issue {issue_number} for {test['name']}") + print(f"Comment added to issue {issue_number} for {test['name']}.") -def process_failures(report_file): +def process_failures(report_file, package): """Process pytest failures and manage GitHub issues.""" failures = load_pytest_results(report_file) if not failures: @@ -91,13 +131,13 @@ def process_failures(report_file): issue_exists = False for issue in existing_issues: if test["name"] in issue["title"]: - add_comment_to_issue(issue["number"], test) + add_comment_to_issue(issue["number"], test, package) issue_exists = True break if not issue_exists: - create_issue(test) + create_issue(test, package) if __name__ == "__main__": args = parse_arguments() - process_failures(args.results_file) + process_failures(args.results_file, args.package) From 3b0100a37e06d99890dafd503ba287b49e695970 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:56:49 -0600 Subject: [PATCH 126/168] Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index c593a21a0..df733c3db 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -151,6 +151,6 @@ jobs: REPO: ${{ github.repository }} run: | cd pkgs/swarmauri # Change to the directory containing pyproject.toml - poetry run python ../../scripts/manage_issues.py --results-file=pytest_results.json + poetry run python ../../scripts/manage_issues.py --results-file=pytest_results.json --package=${{ matrix.package_tests.package }} From 0c4eb4272a8317b6c6237a4d1f501b1c67a4bfcf Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:05:31 -0600 Subject: [PATCH 127/168] cicd - Update manage_issues.py --- scripts/manage_issues.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/manage_issues.py b/scripts/manage_issues.py index b7da50ab2..d08e03acd 100644 --- a/scripts/manage_issues.py +++ b/scripts/manage_issues.py @@ -11,7 +11,7 @@ "Accept": "application/vnd.github.v3+json", } -BASE_BRANCH = os.getenv("GITHUB_REF", "unknown").split("/")[-1] +BASE_BRANCH = os.getenv("GITHUB_HEAD_REF") or os.getenv("GITHUB_REF", "unknown").split("/")[-1] COMMIT_SHA = os.getenv("GITHUB_SHA", "unknown") WORKFLOW_RUN_URL = os.getenv("GITHUB_SERVER_URL", "https://github.com") + f"/{REPO}/actions/runs/{os.getenv('GITHUB_RUN_ID', 'unknown')}" From 358f6ae930bc66a20c67e1e3c79059ee85fc4719 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:15:48 -0600 Subject: [PATCH 128/168] Update manage_issues.py --- scripts/manage_issues.py | 79 +++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 42 deletions(-) diff --git a/scripts/manage_issues.py b/scripts/manage_issues.py index d08e03acd..89cde4393 100644 --- a/scripts/manage_issues.py +++ b/scripts/manage_issues.py @@ -1,6 +1,8 @@ import os import json import requests +from swarmauri.llms.concrete.GroqModel import GroqModel +from swarmauri.agents.concrete.SimpleConversationAgent import SimpleConversationAgent import argparse # GitHub API settings @@ -11,6 +13,11 @@ "Accept": "application/vnd.github.v3+json", } +# GroqModel Initialization +GROQ_API_KEY = os.getenv("GROQ_API_KEY") +llm = GroqModel(api_key=GROQ_API_KEY) +agent = SimpleConversationAgent(llm=llm) + BASE_BRANCH = os.getenv("GITHUB_HEAD_REF") or os.getenv("GITHUB_REF", "unknown").split("/")[-1] COMMIT_SHA = os.getenv("GITHUB_SHA", "unknown") WORKFLOW_RUN_URL = os.getenv("GITHUB_SERVER_URL", "https://github.com") + f"/{REPO}/actions/runs/{os.getenv('GITHUB_RUN_ID', 'unknown')}" @@ -18,18 +25,8 @@ def parse_arguments(): """Parse command-line arguments.""" parser = argparse.ArgumentParser(description="Manage GitHub Issues for Test Failures") - parser.add_argument( - "--results-file", - type=str, - default="pytest_results.json", - help="Path to the pytest JSON report file (default: pytest_results.json)", - ) - parser.add_argument( - "--package", - type=str, - required=True, - help="Name of the matrix package where the error occurred.", - ) + parser.add_argument("--results-file", type=str, required=True, help="Path to the pytest JSON report file") + parser.add_argument("--package", type=str, required=True, help="Name of the matrix package where the error occurred.") return parser.parse_args() def load_pytest_results(report_file): @@ -53,6 +50,24 @@ def load_pytest_results(report_file): }) return failures +def ask_groq_for_fix(test_name, failure_message, stack_trace): + """Ask GroqModel for suggestions on fixing the test failure.""" + prompt = f""" + I have a failing test case named '{test_name}' in a Python project. The error message is: + {failure_message} + + The stack trace is: + {stack_trace} + + Can you help me identify the cause of this failure and suggest a fix? + """ + try: + response = agent.exec(input_str=prompt) + return response + except Exception as e: + print(f"Error communicating with Groq: {e}") + return "Unable to retrieve suggestions from Groq at this time." + def get_existing_issues(): """Retrieve all existing issues with the pytest-failure label.""" url = f"https://api.github.com/repos/{REPO}/issues" @@ -63,9 +78,10 @@ def get_existing_issues(): def create_issue(test, package): """Create a new GitHub issue for the test failure.""" + groq_suggestion = ask_groq_for_fix(test["name"], test["message"], test["message"]) url = f"https://api.github.com/repos/{REPO}/issues" - # Construct the issue body with enhanced documentation + # Construct the issue body data = { "title": f"[Test Case Failure]: {test['name']}", "body": f""" @@ -77,46 +93,25 @@ def create_issue(test, package): --- +### Suggested Fix (via Groq): +{groq_suggestion} + +--- + ### Context: - **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) - **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) - **Workflow Run**: [View Run]({WORKFLOW_RUN_URL}) - **Matrix Package**: `{package}` ---- - -### Label: +### Labels: This issue is auto-labeled for the `{package}` package. """, "labels": ["pytest-failure", package] } response = requests.post(url, headers=HEADERS, json=data) response.raise_for_status() - print(f"Issue created for {test['name']} in package '{package}'.") - -def add_comment_to_issue(issue_number, test, package): - """Add a comment to an existing GitHub issue.""" - url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" - data = {"body": f""" -New failure detected: - -### Test Case: -{test['path']} - -### Details: -{test['message']} - ---- - -### Context: -- **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) -- **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) -- **Workflow Run**: [View Run]({WORKFLOW_RUN_URL}) -- **Matrix Package**: `{package}` -"""} - response = requests.post(url, headers=HEADERS, json=data) - response.raise_for_status() - print(f"Comment added to issue {issue_number} for {test['name']}.") + print(f"Issue created for {test['name']} with Groq suggestion in package '{package}'.") def process_failures(report_file, package): """Process pytest failures and manage GitHub issues.""" @@ -131,7 +126,7 @@ def process_failures(report_file, package): issue_exists = False for issue in existing_issues: if test["name"] in issue["title"]: - add_comment_to_issue(issue["number"], test, package) + print(f"Issue already exists for {test['name']}. Skipping...") issue_exists = True break From 4b4ec5747c586c8beef8bf68626a1d58b0b7bd69 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:21:23 -0600 Subject: [PATCH 129/168] cicd - Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index df733c3db..f5c7efb92 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -147,6 +147,7 @@ jobs: - name: Process test results and manage issues if: always() env: + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} run: | From 94500037118364ab43f616bda7e6a52d7ca48ae0 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:28:02 -0600 Subject: [PATCH 130/168] Update manage_issues.py --- scripts/manage_issues.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/manage_issues.py b/scripts/manage_issues.py index 89cde4393..ddd177ab2 100644 --- a/scripts/manage_issues.py +++ b/scripts/manage_issues.py @@ -3,6 +3,7 @@ import requests from swarmauri.llms.concrete.GroqModel import GroqModel from swarmauri.agents.concrete.SimpleConversationAgent import SimpleConversationAgent +from swarmauri.conversations.concrete.Conversation import Conversation import argparse # GitHub API settings @@ -16,7 +17,6 @@ # GroqModel Initialization GROQ_API_KEY = os.getenv("GROQ_API_KEY") llm = GroqModel(api_key=GROQ_API_KEY) -agent = SimpleConversationAgent(llm=llm) BASE_BRANCH = os.getenv("GITHUB_HEAD_REF") or os.getenv("GITHUB_REF", "unknown").split("/")[-1] COMMIT_SHA = os.getenv("GITHUB_SHA", "unknown") @@ -62,6 +62,7 @@ def ask_groq_for_fix(test_name, failure_message, stack_trace): Can you help me identify the cause of this failure and suggest a fix? """ try: + agent = SimpleConversationAgent(llm=llm, conversation=Conversation()) response = agent.exec(input_str=prompt) return response except Exception as e: From 8130497e220f0a931747337979f3a1de1b368050 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:32:43 -0600 Subject: [PATCH 131/168] Create LazyLoader.py --- pkgs/swarmauri/swarmauri/utils/LazyLoader.py | 31 ++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 pkgs/swarmauri/swarmauri/utils/LazyLoader.py diff --git a/pkgs/swarmauri/swarmauri/utils/LazyLoader.py b/pkgs/swarmauri/swarmauri/utils/LazyLoader.py new file mode 100644 index 000000000..065065861 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/utils/LazyLoader.py @@ -0,0 +1,31 @@ +import importlib + +class LazyLoader: + def __init__(self, module_name, class_name): + self.module_name = module_name + self.class_name = class_name + self._loaded_class = None + + def _load_class(self): + if self._loaded_class is None: + try: + module = importlib.import_module(self.module_name) + self._loaded_class = getattr(module, self.class_name) + except ImportError: + print( + f"Warning: The module '{self.module_name}' is not available. " + f"Please install the necessary dependencies to enable this functionality." + ) + self._loaded_class = None + except AttributeError: + print( + f"Warning: The class '{self.class_name}' was not found in module '{self.module_name}'." + ) + self._loaded_class = None + return self._loaded_class + + def __getattr__(self, item): + loaded_class = self._load_class() + if loaded_class is None: + raise ImportError(f"Unable to load class {self.class_name} from {self.module_name}") + return getattr(loaded_class, item) From aa915b6fc707304713ce8ec8ba0e1ffdb13e9ebb Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:34:42 -0600 Subject: [PATCH 132/168] swarm - shift from _lazy_import to LazyLoader --- pkgs/swarmauri/swarmauri/tools/concrete/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkgs/swarmauri/swarmauri/tools/concrete/__init__.py b/pkgs/swarmauri/swarmauri/tools/concrete/__init__.py index 5b7d61054..173b3ac4b 100644 --- a/pkgs/swarmauri/swarmauri/tools/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/tools/concrete/__init__.py @@ -1,4 +1,4 @@ -from swarmauri.utils._lazy_import import _lazy_import +from swarmauri.utils.LazyLoader import LazyLoader # List of tool names (file names without the ".py" extension) and corresponding class names tool_files = [ @@ -27,9 +27,9 @@ ("swarmauri.tools.concrete.WeatherTool", "WeatherTool"), ] -# Lazy loading of tools, storing them in variables +# Lazy loading of tools using LazyLoader for module_name, class_name in tool_files: - globals()[class_name] = _lazy_import(module_name, class_name) + globals()[class_name] = LazyLoader(module_name, class_name) -# Adding the lazy-loaded tools to __all__ +# Adding tools to __all__ (still safe because LazyLoader doesn't raise errors until accessed) __all__ = [class_name for _, class_name in tool_files] From d15ef7887bfbfe8d33c00b4cfb42c2ef50c7cc89 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:39:18 -0600 Subject: [PATCH 133/168] cicd - Update manage_issues.py --- scripts/manage_issues.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/scripts/manage_issues.py b/scripts/manage_issues.py index ddd177ab2..bab5fb1ad 100644 --- a/scripts/manage_issues.py +++ b/scripts/manage_issues.py @@ -102,6 +102,7 @@ def create_issue(test, package): ### Context: - **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) - **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) +- **Commit Tree**: [{COMMIT_SHA}](https://github.com/{REPO}/tree/{COMMIT_SHA}) - **Workflow Run**: [View Run]({WORKFLOW_RUN_URL}) - **Matrix Package**: `{package}` @@ -114,6 +115,31 @@ def create_issue(test, package): response.raise_for_status() print(f"Issue created for {test['name']} with Groq suggestion in package '{package}'.") +def add_comment_to_issue(issue_number, test, package): + """Add a comment to an existing GitHub issue.""" + url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" + data = {"body": f""" +New failure detected: + +### Test Case: +{test['path']} + +### Details: +{test['message']} + +--- + +### Context: +- **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) +- **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) +- **Commit Tree**: [{COMMIT_SHA}](https://github.com/{REPO}/tree/{COMMIT_SHA}) +- **Workflow Run**: [View Run]({WORKFLOW_RUN_URL}) +- **Matrix Package**: `{package}` +"""} + response = requests.post(url, headers=HEADERS, json=data) + response.raise_for_status() + print(f"Comment added to issue {issue_number} for {test['name']}.") + def process_failures(report_file, package): """Process pytest failures and manage GitHub issues.""" failures = load_pytest_results(report_file) @@ -127,7 +153,7 @@ def process_failures(report_file, package): issue_exists = False for issue in existing_issues: if test["name"] in issue["title"]: - print(f"Issue already exists for {test['name']}. Skipping...") + add_comment_to_issue(issue["number"], test, package) issue_exists = True break From 36817168be4950dcb16df38601e1078457ccf701 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:56:45 -0600 Subject: [PATCH 134/168] Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index 624a74b0e..18ea3f273 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -49,7 +49,7 @@ yake = { version = "==0.4.8", optional = true } beautifulsoup4 = { version = "04.12.3", optional = true } #gensim = { version = "==4.3.3", optional = true } scipy = { version = ">=1.7.0,<1.14.0", optional = true } -#scikit-learn = { version = "^1.4.2", optional = true } +scikit-learn = { version = "^1.4.2", optional = true } #spacy = { version = ">=3.0.0,<=3.8.2", optional = true } #transformers = { version = "^4.45.0", optional = true } #torch = { version = "^2.5.0", optional = true } @@ -68,6 +68,7 @@ nlp = [ ] nlp_tools = ["beautifulsoup4"] #ml_toolkits = ["gensim", "scipy", "scikit-learn"] +ml_toolkits = ["scikit-learn"] #spacy = ["spacy"] #transformers = ["transformers"] #torch = ["torch"] @@ -81,6 +82,7 @@ full = [ #"nltk", "textblob", "yake", "beautifulsoup4", + "scikit-learn", #"gensim", "scipy", "scikit-learn", #"spacy", #"transformers", From 45a416f930175b3cf05c16511b3e0890f805d26c Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:59:52 -0600 Subject: [PATCH 135/168] Create rag_issue_manager.py --- scripts/rag_issue_manager.py | 178 +++++++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 scripts/rag_issue_manager.py diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py new file mode 100644 index 000000000..0605fa690 --- /dev/null +++ b/scripts/rag_issue_manager.py @@ -0,0 +1,178 @@ +import os +import json +import requests +from swarmauri.utils.load_documents_from_folder import load_documents_from_folder +from swarmauri.llms.concrete.GroqModel import GroqModel +from swarmauri.agents.concrete.RagAgent import RagAgent +from swarmauri.conversations.concrete.Conversation import Conversation +from swarmauri.vector_stores.concrete.TfidfVectorStore import TfidfVectorStore +import argparse + +# GitHub API settings +GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") +REPO = os.getenv("REPO") +HEADERS = { + "Authorization": f"Bearer {GITHUB_TOKEN}", + "Accept": "application/vnd.github.v3+json", +} + +# GroqModel Initialization +GROQ_API_KEY = os.getenv("GROQ_API_KEY") +llm = GroqModel(api_key=GROQ_API_KEY) + +BASE_BRANCH = os.getenv("GITHUB_HEAD_REF") or os.getenv("GITHUB_REF", "unknown").split("/")[-1] +COMMIT_SHA = os.getenv("GITHUB_SHA", "unknown") +WORKFLOW_RUN_URL = os.getenv("GITHUB_SERVER_URL", "https://github.com") + f"/{REPO}/actions/runs/{os.getenv('GITHUB_RUN_ID', 'unknown')}" + +def parse_arguments(): + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description="Manage GitHub Issues for Test Failures") + parser.add_argument("--results-file", type=str, required=True, help="Path to the pytest JSON report file") + parser.add_argument("--package", type=str, required=True, help="Name of the matrix package where the error occurred.") + return parser.parse_args() + +def load_pytest_results(report_file): + """Load pytest results from the JSON report.""" + if not os.path.exists(report_file): + print(f"Report file not found: {report_file}") + return [] + + with open(report_file, "r") as f: + report = json.load(f) + + failures = [] + for test in report.get("tests", []): + if test.get("outcome") == "failed": + test_name = test.get("nodeid", "Unknown test") + failure_message = test.get("call", {}).get("longrepr", "No failure details available") + failures.append({ + "name": test_name, + "path": test.get("nodeid", "Unknown path"), + "message": failure_message + }) + return failures + +def ask_groq_for_fix(test_name, failure_message, stack_trace): + """Ask GroqModel for suggestions on fixing the test failure.""" + prompt = f""" + I have a failing test case named '{test_name}' in a Python project. The error message is: + {failure_message} + + The stack trace is: + {stack_trace} + + Can you help me identify the cause of this failure and suggest a fix? + """ + try: + current_directory = os.getcwd() + documents = load_documents_from_folder(folder_path=current_directory) + print(f"Loaded {len(documents)} documents.") + + # Step 3: Initialize the TFIDF Vector Store and add documents + vector_store = TfidfVectorStore() + vector_store.add_documents(documents) + print("Documents have been added to the TFIDF Vector Store.") + + # Step 5: Initialize the RagAgent with the vector store and language model + rag_agent = RagAgent(llm=llm, vector_store=vector_store, conversation=Conversation() top_k=20) + print("RagAgent initialized successfully.") + response = agent.exec(input_str=prompt) + return response + except Exception as e: + print(f"Error communicating with Groq: {e}") + return "Unable to retrieve suggestions from Groq at this time." + +def get_existing_issues(): + """Retrieve all existing issues with the pytest-failure label.""" + url = f"https://api.github.com/repos/{REPO}/issues" + params = {"labels": "pytest-failure", "state": "open"} + response = requests.get(url, headers=HEADERS, params=params) + response.raise_for_status() + return response.json() + +def create_issue(test, package): + """Create a new GitHub issue for the test failure.""" + groq_suggestion = ask_groq_for_fix(test["name"], test["message"], test["message"]) + url = f"https://api.github.com/repos/{REPO}/issues" + + # Construct the issue body + data = { + "title": f"[Test Case Failure]: {test['name']}", + "body": f""" +### Test Case: +{test['path']} + +### Failure Details: +{test['message']} + +--- + +### Suggested Fix (via Groq): +{groq_suggestion} + +--- + +### Context: +- **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) +- **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) +- **Commit Tree**: [{COMMIT_SHA}](https://github.com/{REPO}/tree/{COMMIT_SHA}) +- **Workflow Run**: [View Run]({WORKFLOW_RUN_URL}) +- **Matrix Package**: `{package}` + +### Labels: +This issue is auto-labeled for the `{package}` package. +""", + "labels": ["pytest-failure", package] + } + response = requests.post(url, headers=HEADERS, json=data) + response.raise_for_status() + print(f"Issue created for {test['name']} with Groq suggestion in package '{package}'.") + +def add_comment_to_issue(issue_number, test, package): + """Add a comment to an existing GitHub issue.""" + url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" + data = {"body": f""" +New failure detected: + +### Test Case: +{test['path']} + +### Details: +{test['message']} + +--- + +### Context: +- **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) +- **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) +- **Commit Tree**: [{COMMIT_SHA}](https://github.com/{REPO}/tree/{COMMIT_SHA}) +- **Workflow Run**: [View Run]({WORKFLOW_RUN_URL}) +- **Matrix Package**: `{package}` +"""} + response = requests.post(url, headers=HEADERS, json=data) + response.raise_for_status() + print(f"Comment added to issue {issue_number} for {test['name']}.") + +def process_failures(report_file, package): + """Process pytest failures and manage GitHub issues.""" + failures = load_pytest_results(report_file) + if not failures: + print("No test failures found.") + return + + existing_issues = get_existing_issues() + + for test in failures: + issue_exists = False + for issue in existing_issues: + if test["name"] in issue["title"]: + add_comment_to_issue(issue["number"], test, package) + issue_exists = True + break + + if not issue_exists: + create_issue(test, package) + +if __name__ == "__main__": + args = parse_arguments() + process_failures(args.results_file, args.package) From 7a1cabd1622849f44f9ee6179f4b78e187f1384e Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:05:45 -0600 Subject: [PATCH 136/168] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 0605fa690..3541e5803 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -54,7 +54,9 @@ def load_pytest_results(report_file): def ask_groq_for_fix(test_name, failure_message, stack_trace): """Ask GroqModel for suggestions on fixing the test failure.""" + system_context = f"Utilizing the following codebase solve the user's problem.\nCodebase:" prompt = f""" + \n\nUser Problem: I have a failing test case named '{test_name}' in a Python project. The error message is: {failure_message} @@ -74,9 +76,10 @@ def ask_groq_for_fix(test_name, failure_message, stack_trace): print("Documents have been added to the TFIDF Vector Store.") # Step 5: Initialize the RagAgent with the vector store and language model - rag_agent = RagAgent(llm=llm, vector_store=vector_store, conversation=Conversation() top_k=20) + rag_agent = RagAgent(system_context=system_context, + llm=llm, vector_store=vector_store, conversation=Conversation(), top_k=20) print("RagAgent initialized successfully.") - response = agent.exec(input_str=prompt) + response = rag_agent.exec(input_str=prompt, llm_kwargs={"max_tokens": 1750, "temperature":0.7}) return response except Exception as e: print(f"Error communicating with Groq: {e}") From 0fc76bc2e39d07756af9b24edf6517c78e74cbe1 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:06:30 -0600 Subject: [PATCH 137/168] cicd - Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index f5c7efb92..b1c1c76a7 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -152,6 +152,6 @@ jobs: REPO: ${{ github.repository }} run: | cd pkgs/swarmauri # Change to the directory containing pyproject.toml - poetry run python ../../scripts/manage_issues.py --results-file=pytest_results.json --package=${{ matrix.package_tests.package }} + poetry run python ../../scripts/rag_issue_manager.py --results-file=pytest_results.json --package=${{ matrix.package_tests.package }} From 171836a3d2b8bec1b9fb487f0f7fea1e50297429 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:13:22 -0600 Subject: [PATCH 138/168] cicd - Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 3541e5803..827a60bc7 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -16,14 +16,15 @@ "Accept": "application/vnd.github.v3+json", } -# GroqModel Initialization -GROQ_API_KEY = os.getenv("GROQ_API_KEY") -llm = GroqModel(api_key=GROQ_API_KEY) - +# GitHub Metadata BASE_BRANCH = os.getenv("GITHUB_HEAD_REF") or os.getenv("GITHUB_REF", "unknown").split("/")[-1] COMMIT_SHA = os.getenv("GITHUB_SHA", "unknown") WORKFLOW_RUN_URL = os.getenv("GITHUB_SERVER_URL", "https://github.com") + f"/{REPO}/actions/runs/{os.getenv('GITHUB_RUN_ID', 'unknown')}" +# GroqModel Initialization +GROQ_API_KEY = os.getenv("GROQ_API_KEY") +llm = GroqModel(api_key=GROQ_API_KEY) + def parse_arguments(): """Parse command-line arguments.""" parser = argparse.ArgumentParser(description="Manage GitHub Issues for Test Failures") From fa838085302aa8f1f6a745ed93750a86a79696e8 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:16:44 -0600 Subject: [PATCH 139/168] cicd - Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 827a60bc7..5c5d004a5 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -68,7 +68,7 @@ def ask_groq_for_fix(test_name, failure_message, stack_trace): """ try: current_directory = os.getcwd() - documents = load_documents_from_folder(folder_path=current_directory) + documents = load_documents_from_folder(folder_path=current_directory, include_extensions=['.py', '.md', '.yaml', '.toml', '.json']) print(f"Loaded {len(documents)} documents.") # Step 3: Initialize the TFIDF Vector Store and add documents @@ -134,6 +134,7 @@ def create_issue(test, package): def add_comment_to_issue(issue_number, test, package): """Add a comment to an existing GitHub issue.""" + groq_suggestion = ask_groq_for_fix(test["name"], test["message"], test["message"]) url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" data = {"body": f""" New failure detected: @@ -141,11 +142,16 @@ def add_comment_to_issue(issue_number, test, package): ### Test Case: {test['path']} -### Details: +### Failure Details: {test['message']} --- +### Suggested Fix (via Groq): +{groq_suggestion} + +--- + ### Context: - **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) - **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) From 5591fa28e08a8103770ee580637f0ac75bad263b Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:19:03 -0600 Subject: [PATCH 140/168] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 5c5d004a5..9062768bd 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -68,7 +68,8 @@ def ask_groq_for_fix(test_name, failure_message, stack_trace): """ try: current_directory = os.getcwd() - documents = load_documents_from_folder(folder_path=current_directory, include_extensions=['.py', '.md', '.yaml', '.toml', '.json']) + documents = load_documents_from_folder(folder_path=current_directory, + include_extensions=['py', 'md', 'yaml', 'toml', 'json']) print(f"Loaded {len(documents)} documents.") # Step 3: Initialize the TFIDF Vector Store and add documents From 2d1223ec8d9c84e897e5b3d2fe33f8dedce679ef Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:21:12 -0600 Subject: [PATCH 141/168] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 9062768bd..cbfc098ce 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -79,9 +79,9 @@ def ask_groq_for_fix(test_name, failure_message, stack_trace): # Step 5: Initialize the RagAgent with the vector store and language model rag_agent = RagAgent(system_context=system_context, - llm=llm, vector_store=vector_store, conversation=Conversation(), top_k=20) + llm=llm, vector_store=vector_store, conversation=Conversation()) print("RagAgent initialized successfully.") - response = rag_agent.exec(input_str=prompt, llm_kwargs={"max_tokens": 1750, "temperature":0.7}) + response = rag_agent.exec(input_str=prompt, top_k=20, llm_kwargs={"max_tokens": 1750, "temperature":0.7}) return response except Exception as e: print(f"Error communicating with Groq: {e}") From e8c65a590372cc3882d893af36a32ba2f9bcc9c7 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:28:02 -0600 Subject: [PATCH 142/168] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index cbfc098ce..9393ffbb4 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -2,7 +2,7 @@ import json import requests from swarmauri.utils.load_documents_from_folder import load_documents_from_folder -from swarmauri.llms.concrete.GroqModel import GroqModel +from swarmauri.llms.concrete.DeepInfraModel import DeepInfraModel from swarmauri.agents.concrete.RagAgent import RagAgent from swarmauri.conversations.concrete.Conversation import Conversation from swarmauri.vector_stores.concrete.TfidfVectorStore import TfidfVectorStore @@ -21,9 +21,9 @@ COMMIT_SHA = os.getenv("GITHUB_SHA", "unknown") WORKFLOW_RUN_URL = os.getenv("GITHUB_SERVER_URL", "https://github.com") + f"/{REPO}/actions/runs/{os.getenv('GITHUB_RUN_ID', 'unknown')}" -# GroqModel Initialization -GROQ_API_KEY = os.getenv("GROQ_API_KEY") -llm = GroqModel(api_key=GROQ_API_KEY) +# DeepInfraModel Initialization +DEEPINFRA_API_KEY = os.getenv("DEEPINFRA_API_KEY") +llm = DeepInfraModel(api_key=DEEPINFRA_API_KEY, name="meta-llama/Meta-Llama-3.1-405B-Instruct") def parse_arguments(): """Parse command-line arguments.""" From 16b5ec9a57b6e731a6429dcde9e958b164fa9ffa Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:28:26 -0600 Subject: [PATCH 143/168] Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index b1c1c76a7..8cdc38823 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -147,7 +147,7 @@ jobs: - name: Process test results and manage issues if: always() env: - GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} + DEEPINFRA_API_KEY: ${{ secrets.DEEPINFRA_API_KEY }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} run: | From 0d186841bd976aaeba15deda08a7a0635d15b41c Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:29:42 -0600 Subject: [PATCH 144/168] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 9393ffbb4..cfdde97ca 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -81,7 +81,7 @@ def ask_groq_for_fix(test_name, failure_message, stack_trace): rag_agent = RagAgent(system_context=system_context, llm=llm, vector_store=vector_store, conversation=Conversation()) print("RagAgent initialized successfully.") - response = rag_agent.exec(input_str=prompt, top_k=20, llm_kwargs={"max_tokens": 1750, "temperature":0.7}) + response = rag_agent.exec(input_data=prompt, top_k=20, llm_kwargs={"max_tokens": 1750, "temperature":0.7}) return response except Exception as e: print(f"Error communicating with Groq: {e}") From 845fce535fdeef5dcf00eed55a79b4891786d35e Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:31:20 -0600 Subject: [PATCH 145/168] Update SimpleConversationAgent.py --- .../swarmauri/agents/concrete/SimpleConversationAgent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/swarmauri/agents/concrete/SimpleConversationAgent.py b/pkgs/swarmauri/swarmauri/agents/concrete/SimpleConversationAgent.py index 9653cf9c6..3dcd0d4bf 100644 --- a/pkgs/swarmauri/swarmauri/agents/concrete/SimpleConversationAgent.py +++ b/pkgs/swarmauri/swarmauri/agents/concrete/SimpleConversationAgent.py @@ -16,7 +16,7 @@ class SimpleConversationAgent(AgentConversationMixin, AgentBase): def exec( self, - input_str: Optional[Union[str, List[contentItem]]] = "", + input_data: Optional[Union[str, List[contentItem]]] = "", llm_kwargs: Optional[Dict] = {}, ) -> Any: From 7065cc1871f6edcd2748dbc2b50efd10af50c90e Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:32:37 -0600 Subject: [PATCH 146/168] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index cfdde97ca..8fc3908a7 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -4,7 +4,7 @@ from swarmauri.utils.load_documents_from_folder import load_documents_from_folder from swarmauri.llms.concrete.DeepInfraModel import DeepInfraModel from swarmauri.agents.concrete.RagAgent import RagAgent -from swarmauri.conversations.concrete.Conversation import Conversation +from swarmauri.conversations.concrete.SystemContextConversation import SystemContextConversation from swarmauri.vector_stores.concrete.TfidfVectorStore import TfidfVectorStore import argparse @@ -79,7 +79,7 @@ def ask_groq_for_fix(test_name, failure_message, stack_trace): # Step 5: Initialize the RagAgent with the vector store and language model rag_agent = RagAgent(system_context=system_context, - llm=llm, vector_store=vector_store, conversation=Conversation()) + llm=llm, vector_store=vector_store, conversation=SystemContextConversation()) print("RagAgent initialized successfully.") response = rag_agent.exec(input_data=prompt, top_k=20, llm_kwargs={"max_tokens": 1750, "temperature":0.7}) return response From 81440662dab0ad92a553c136a8d787dbad3a09a3 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:34:38 -0600 Subject: [PATCH 147/168] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 8fc3908a7..81d3a1c5b 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -4,7 +4,7 @@ from swarmauri.utils.load_documents_from_folder import load_documents_from_folder from swarmauri.llms.concrete.DeepInfraModel import DeepInfraModel from swarmauri.agents.concrete.RagAgent import RagAgent -from swarmauri.conversations.concrete.SystemContextConversation import SystemContextConversation +from swarmauri.conversations.concrete.MaxSystemContextConversation import MaxSystemContextConversation from swarmauri.vector_stores.concrete.TfidfVectorStore import TfidfVectorStore import argparse @@ -79,7 +79,7 @@ def ask_groq_for_fix(test_name, failure_message, stack_trace): # Step 5: Initialize the RagAgent with the vector store and language model rag_agent = RagAgent(system_context=system_context, - llm=llm, vector_store=vector_store, conversation=SystemContextConversation()) + llm=llm, vector_store=vector_store, conversation=MaxSystemContextConversation()) print("RagAgent initialized successfully.") response = rag_agent.exec(input_data=prompt, top_k=20, llm_kwargs={"max_tokens": 1750, "temperature":0.7}) return response From d59c1ec4bb239bb838c5bf1f28564b3e38ab6aa0 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:40:00 -0600 Subject: [PATCH 148/168] cicd - Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 81d3a1c5b..a527a65ec 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -81,7 +81,8 @@ def ask_groq_for_fix(test_name, failure_message, stack_trace): rag_agent = RagAgent(system_context=system_context, llm=llm, vector_store=vector_store, conversation=MaxSystemContextConversation()) print("RagAgent initialized successfully.") - response = rag_agent.exec(input_data=prompt, top_k=20, llm_kwargs={"max_tokens": 1750, "temperature":0.7}) + response = rag_agent.exec(input_data=prompt, top_k=20, llm_kwargs={"max_tokens": 750, "temperature":0.7}) + print(f"\nPrompt: \n{prompt}\n\n", '-'*10, f"Response: {response}\n\n", '='*10, '\n') return response except Exception as e: print(f"Error communicating with Groq: {e}") From f28cdd3d2204f7aa64b7b30dbff84be5aa5ce31e Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:42:11 -0600 Subject: [PATCH 149/168] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index a527a65ec..71e40c9e9 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -53,8 +53,8 @@ def load_pytest_results(report_file): }) return failures -def ask_groq_for_fix(test_name, failure_message, stack_trace): - """Ask GroqModel for suggestions on fixing the test failure.""" +def ask_agent_for_fix(test_name, failure_message, stack_trace): + """Ask the LLM Model for suggestions on fixing the test failure.""" system_context = f"Utilizing the following codebase solve the user's problem.\nCodebase:" prompt = f""" \n\nUser Problem: @@ -85,8 +85,8 @@ def ask_groq_for_fix(test_name, failure_message, stack_trace): print(f"\nPrompt: \n{prompt}\n\n", '-'*10, f"Response: {response}\n\n", '='*10, '\n') return response except Exception as e: - print(f"Error communicating with Groq: {e}") - return "Unable to retrieve suggestions from Groq at this time." + print(f"Error communicating with LLM: {e}") + return "Unable to retrieve suggestions from LLM at this time." def get_existing_issues(): """Retrieve all existing issues with the pytest-failure label.""" @@ -98,7 +98,7 @@ def get_existing_issues(): def create_issue(test, package): """Create a new GitHub issue for the test failure.""" - groq_suggestion = ask_groq_for_fix(test["name"], test["message"], test["message"]) + suggestion = ask_agent_for_fix(test["name"], test["message"], test["message"]) url = f"https://api.github.com/repos/{REPO}/issues" # Construct the issue body @@ -113,8 +113,8 @@ def create_issue(test, package): --- -### Suggested Fix (via Groq): -{groq_suggestion} +### Suggested Fix (via Agent): +{suggestion} --- @@ -132,11 +132,11 @@ def create_issue(test, package): } response = requests.post(url, headers=HEADERS, json=data) response.raise_for_status() - print(f"Issue created for {test['name']} with Groq suggestion in package '{package}'.") + print(f"Issue created for {test['name']} with LLM suggestion in package '{package}'.") def add_comment_to_issue(issue_number, test, package): """Add a comment to an existing GitHub issue.""" - groq_suggestion = ask_groq_for_fix(test["name"], test["message"], test["message"]) + suggestion = ask_agent_for_fix(test["name"], test["message"], test["message"]) url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" data = {"body": f""" New failure detected: @@ -149,8 +149,8 @@ def add_comment_to_issue(issue_number, test, package): --- -### Suggested Fix (via Groq): -{groq_suggestion} +### Suggested Fix (via Agent): +{suggestion} --- From b8cb8acbf0f45fc2326bfe53adfbca9333d4a21c Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:02:41 -0600 Subject: [PATCH 150/168] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 71e40c9e9..ab0a7d753 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -100,6 +100,11 @@ def create_issue(test, package): """Create a new GitHub issue for the test failure.""" suggestion = ask_agent_for_fix(test["name"], test["message"], test["message"]) url = f"https://api.github.com/repos/{REPO}/issues" + if package == 'core' or 'community' or 'experimental': + package_name = f"swarmauri_{package}" + resource_kind, type_kind = test['name'].split('/')[2:4] + comp_file_url = f"pkgs/{package}/{package_name}/{resource_kind}/concrete/{type_kind}.py" + test_file_url = f"pkgs/{package}/{test['name']}" # Construct the issue body data = { @@ -119,6 +124,8 @@ def create_issue(test, package): --- ### Context: +- **Component**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{comp_file_url) +- **Test File**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{test_file_url) - **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) - **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) - **Commit Tree**: [{COMMIT_SHA}](https://github.com/{REPO}/tree/{COMMIT_SHA}) @@ -138,6 +145,11 @@ def add_comment_to_issue(issue_number, test, package): """Add a comment to an existing GitHub issue.""" suggestion = ask_agent_for_fix(test["name"], test["message"], test["message"]) url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" + if package == 'core' or 'community' or 'experimental': + package_name = f"swarmauri_{package}" + resource_kind, type_kind = test['name'].split('/')[2:4] + comp_file_url = f"pkgs/{package}/{package_name}/{resource_kind}/concrete/{type_kind}.py" + test_file_url = f"pkgs/{package}/{test['name']}" data = {"body": f""" New failure detected: @@ -155,6 +167,8 @@ def add_comment_to_issue(issue_number, test, package): --- ### Context: +- **Component**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{comp_file_url) +- **Test File**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{test_file_url) - **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) - **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) - **Commit Tree**: [{COMMIT_SHA}](https://github.com/{REPO}/tree/{COMMIT_SHA}) From 37b148f270e12e671e5a550a049fe269f06ed626 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:03:55 -0600 Subject: [PATCH 151/168] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index ab0a7d753..e8e56715e 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -124,8 +124,8 @@ def create_issue(test, package): --- ### Context: -- **Component**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{comp_file_url) -- **Test File**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{test_file_url) +- **Component**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{comp_file_url}) +- **Test File**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{test_file_url}) - **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) - **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) - **Commit Tree**: [{COMMIT_SHA}](https://github.com/{REPO}/tree/{COMMIT_SHA}) @@ -167,8 +167,8 @@ def add_comment_to_issue(issue_number, test, package): --- ### Context: -- **Component**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{comp_file_url) -- **Test File**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{test_file_url) +- **Component**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{comp_file_url}) +- **Test File**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{test_file_url}) - **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) - **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) - **Commit Tree**: [{COMMIT_SHA}](https://github.com/{REPO}/tree/{COMMIT_SHA}) From 9e72c45dd369105b6ddcfc3295fbeb48889d546f Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:11:08 -0600 Subject: [PATCH 152/168] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index e8e56715e..e3a5408c9 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -103,8 +103,8 @@ def create_issue(test, package): if package == 'core' or 'community' or 'experimental': package_name = f"swarmauri_{package}" resource_kind, type_kind = test['name'].split('/')[2:4] - comp_file_url = f"pkgs/{package}/{package_name}/{resource_kind}/concrete/{type_kind}.py" - test_file_url = f"pkgs/{package}/{test['name']}" + comp_file_url = f"pkgs/{package}/{package_name}/{resource_kind}/concrete/{type_kind.split('_')[0]}.py" + test_file_url = f"pkgs/{package}/{test['name'].split('::')[0]}" # Construct the issue body data = { @@ -147,9 +147,9 @@ def add_comment_to_issue(issue_number, test, package): url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" if package == 'core' or 'community' or 'experimental': package_name = f"swarmauri_{package}" - resource_kind, type_kind = test['name'].split('/')[2:4] - comp_file_url = f"pkgs/{package}/{package_name}/{resource_kind}/concrete/{type_kind}.py" - test_file_url = f"pkgs/{package}/{test['name']}" + resource_kind, type_kinds = test['name'].split('/')[2:4] + comp_file_url = f"pkgs/{package}/{package_name}/{resource_kind}/concrete/{type_kind.split('_')[0]}.py" + test_file_url = f"pkgs/{package}/{test['name'].split('::')[0]}" data = {"body": f""" New failure detected: From fb04556044e9a5ce349c7f41052728919b0d36e8 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:13:56 -0600 Subject: [PATCH 153/168] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index e3a5408c9..3a8d7f4f9 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -147,7 +147,7 @@ def add_comment_to_issue(issue_number, test, package): url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" if package == 'core' or 'community' or 'experimental': package_name = f"swarmauri_{package}" - resource_kind, type_kinds = test['name'].split('/')[2:4] + resource_kind, type_kind = test['name'].split('/')[2:4] comp_file_url = f"pkgs/{package}/{package_name}/{resource_kind}/concrete/{type_kind.split('_')[0]}.py" test_file_url = f"pkgs/{package}/{test['name'].split('::')[0]}" data = {"body": f""" From 951706ce965af6e318fbd7159bfc413725de0b81 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:15:45 -0600 Subject: [PATCH 154/168] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 3a8d7f4f9..41708a7a6 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -55,10 +55,10 @@ def load_pytest_results(report_file): def ask_agent_for_fix(test_name, failure_message, stack_trace): """Ask the LLM Model for suggestions on fixing the test failure.""" - system_context = f"Utilizing the following codebase solve the user's problem.\nCodebase:" + system_context = f"Utilizing the following codebase solve the user's problem.\nCodebase:\n\n" prompt = f""" \n\nUser Problem: - I have a failing test case named '{test_name}' in a Python project. The error message is: + I have a failing pytest test case named '{test_name}' in a Python project. The error message is: {failure_message} The stack trace is: From 3a7467f78c92822b9da6ead4f2852eb13eb27540 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:18:02 -0600 Subject: [PATCH 155/168] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 41708a7a6..ccb4721d7 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -102,6 +102,8 @@ def create_issue(test, package): url = f"https://api.github.com/repos/{REPO}/issues" if package == 'core' or 'community' or 'experimental': package_name = f"swarmauri_{package}" + else: + package_name = "swarmauri" resource_kind, type_kind = test['name'].split('/')[2:4] comp_file_url = f"pkgs/{package}/{package_name}/{resource_kind}/concrete/{type_kind.split('_')[0]}.py" test_file_url = f"pkgs/{package}/{test['name'].split('::')[0]}" @@ -146,7 +148,9 @@ def add_comment_to_issue(issue_number, test, package): suggestion = ask_agent_for_fix(test["name"], test["message"], test["message"]) url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" if package == 'core' or 'community' or 'experimental': - package_name = f"swarmauri_{package}" + package_name = f"swarmauri_{package}" + else: + package_name = "swarmauri" resource_kind, type_kind = test['name'].split('/')[2:4] comp_file_url = f"pkgs/{package}/{package_name}/{resource_kind}/concrete/{type_kind.split('_')[0]}.py" test_file_url = f"pkgs/{package}/{test['name'].split('::')[0]}" From 832e3669b9b3fbeacf9fb628d6615225ece983c6 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:21:54 -0600 Subject: [PATCH 156/168] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index ccb4721d7..2bbde6d5c 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -100,7 +100,7 @@ def create_issue(test, package): """Create a new GitHub issue for the test failure.""" suggestion = ask_agent_for_fix(test["name"], test["message"], test["message"]) url = f"https://api.github.com/repos/{REPO}/issues" - if package == 'core' or 'community' or 'experimental': + if package in {'core', 'community', 'experimental'}: package_name = f"swarmauri_{package}" else: package_name = "swarmauri" @@ -147,7 +147,7 @@ def add_comment_to_issue(issue_number, test, package): """Add a comment to an existing GitHub issue.""" suggestion = ask_agent_for_fix(test["name"], test["message"], test["message"]) url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" - if package == 'core' or 'community' or 'experimental': + if package in {'core', 'community', 'experimental'}: package_name = f"swarmauri_{package}" else: package_name = "swarmauri" From 24ae1f0b45ef1c676a7cd695fe7bba8b53beee32 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:28:21 -0600 Subject: [PATCH 157/168] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 2bbde6d5c..9e803698e 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -126,13 +126,13 @@ def create_issue(test, package): --- ### Context: -- **Component**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{comp_file_url}) -- **Test File**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{test_file_url}) +- **Matrix Package**: [{package}](https://github.com/{REPO}/tree/{BASE_BRANCH}/pkgs/{package}) +- **Component**: [{comp_file_url}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{comp_file_url}) +- **Test File**: [{test_file_url}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{test_file_url}) - **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) - **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) - **Commit Tree**: [{COMMIT_SHA}](https://github.com/{REPO}/tree/{COMMIT_SHA}) - **Workflow Run**: [View Run]({WORKFLOW_RUN_URL}) -- **Matrix Package**: `{package}` ### Labels: This issue is auto-labeled for the `{package}` package. @@ -171,13 +171,14 @@ def add_comment_to_issue(issue_number, test, package): --- ### Context: -- **Component**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{comp_file_url}) -- **Test File**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{test_file_url}) +- **Matrix Package**: [{package}](https://github.com/{REPO}/tree/{BASE_BRANCH}/pkgs/{package}) +- **Component**: [{comp_file_url}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{comp_file_url}) +- **Test File**: [{test_file_url}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{test_file_url}) - **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) - **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) - **Commit Tree**: [{COMMIT_SHA}](https://github.com/{REPO}/tree/{COMMIT_SHA}) - **Workflow Run**: [View Run]({WORKFLOW_RUN_URL}) -- **Matrix Package**: `{package}` + """} response = requests.post(url, headers=HEADERS, json=data) response.raise_for_status() From 54b6e77c27409ff0183179e180f0ad0d49161a0b Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:30:16 -0600 Subject: [PATCH 158/168] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 9e803698e..683c020eb 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -113,7 +113,7 @@ def create_issue(test, package): "title": f"[Test Case Failure]: {test['name']}", "body": f""" ### Test Case: -{test['path']} +`{test['path']}` ### Failure Details: {test['message']} @@ -158,7 +158,7 @@ def add_comment_to_issue(issue_number, test, package): New failure detected: ### Test Case: -{test['path']} +`{test['path']}` ### Failure Details: {test['message']} From 32b599d7afdef86b83c2a0a25e371ce254e08894 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:36:10 -0600 Subject: [PATCH 159/168] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 683c020eb..3581fecbf 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -116,7 +116,9 @@ def create_issue(test, package): `{test['path']}` ### Failure Details: +```python {test['message']} +``` --- @@ -161,7 +163,9 @@ def add_comment_to_issue(issue_number, test, package): `{test['path']}` ### Failure Details: +```python {test['message']} +``` --- From 41bd7062afa6151bd0fa44b869b2230b413be4e0 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 19:08:49 -0600 Subject: [PATCH 160/168] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 3581fecbf..611adaa41 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -17,7 +17,8 @@ } # GitHub Metadata -BASE_BRANCH = os.getenv("GITHUB_HEAD_REF") or os.getenv("GITHUB_REF", "unknown").split("/")[-1] +BASE_BRANCH = os.getenv("GITHUB_BASE_REF", "unknown") +HEAD_BRANCH = os.getenv("GITHUB_HEAD_REF") or os.getenv("GITHUB_REF", "unknown").split("/")[-1] COMMIT_SHA = os.getenv("GITHUB_SHA", "unknown") WORKFLOW_RUN_URL = os.getenv("GITHUB_SERVER_URL", "https://github.com") + f"/{REPO}/actions/runs/{os.getenv('GITHUB_RUN_ID', 'unknown')}" @@ -128,12 +129,13 @@ def create_issue(test, package): --- ### Context: -- **Matrix Package**: [{package}](https://github.com/{REPO}/tree/{BASE_BRANCH}/pkgs/{package}) -- **Component**: [{comp_file_url}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{comp_file_url}) -- **Test File**: [{test_file_url}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{test_file_url}) -- **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) -- **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) -- **Commit Tree**: [{COMMIT_SHA}](https://github.com/{REPO}/tree/{COMMIT_SHA}) +- **Commit**: `{COMMIT_SHA}` +- **Matrix Package**: [{package}](https://github.com/{REPO}/tree/{HEAD_BRANCH}/pkgs/{package}) +- **Component**: [{comp_file_url}](https://github.com/{REPO}/tree/{HEAD_BRANCH}/{comp_file_url}) +- **Test File**: [{test_file_url}](https://github.com/{REPO}/tree/{HEAD_BRANCH}/{test_file_url}) +- **Branch**: [{HEAD_BRANCH}](https://github.com/{REPO}/tree/{HEAD_BRANCH}) => [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) +- **Changes**: [View Changes](https://github.com/{REPO}/commit/{COMMIT_SHA}) +- **Files**: [View Files](https://github.com/{REPO}/tree/{COMMIT_SHA}) - **Workflow Run**: [View Run]({WORKFLOW_RUN_URL}) ### Labels: @@ -175,12 +177,13 @@ def add_comment_to_issue(issue_number, test, package): --- ### Context: -- **Matrix Package**: [{package}](https://github.com/{REPO}/tree/{BASE_BRANCH}/pkgs/{package}) -- **Component**: [{comp_file_url}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{comp_file_url}) -- **Test File**: [{test_file_url}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{test_file_url}) -- **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) -- **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) -- **Commit Tree**: [{COMMIT_SHA}](https://github.com/{REPO}/tree/{COMMIT_SHA}) +- **Commit**: `{COMMIT_SHA}` +- **Matrix Package**: [{package}](https://github.com/{REPO}/tree/{HEAD_BRANCH}/pkgs/{package}) +- **Component**: [{comp_file_url}](https://github.com/{REPO}/tree/{HEAD_BRANCH}/{comp_file_url}) +- **Test File**: [{test_file_url}](https://github.com/{REPO}/tree/{HEAD_BRANCH}/{test_file_url}) +- **Branch**: [{HEAD_BRANCH}](https://github.com/{REPO}/tree/{HEAD_BRANCH}) => [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) +- **Changes**: [View Changes](https://github.com/{REPO}/commit/{COMMIT_SHA}) +- **Files**: [View Files](https://github.com/{REPO}/tree/{COMMIT_SHA}) - **Workflow Run**: [View Run]({WORKFLOW_RUN_URL}) """} From e33573894d110b65c0e4feab3137ec3f58527121 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 19:13:00 -0600 Subject: [PATCH 161/168] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 611adaa41..4bec5f4ed 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -60,11 +60,11 @@ def ask_agent_for_fix(test_name, failure_message, stack_trace): prompt = f""" \n\nUser Problem: I have a failing pytest test case named '{test_name}' in a Python project. The error message is: - {failure_message} - + `{failure_message}` + \n\n The stack trace is: - {stack_trace} - + `{stack_trace}` + \n\n Can you help me identify the cause of this failure and suggest a fix? """ try: From a9c3c0f67f3fcf5db5c7c5d049bd94c7898633ba Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 19:14:00 -0600 Subject: [PATCH 162/168] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 4bec5f4ed..07d1c3a8b 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -133,7 +133,7 @@ def create_issue(test, package): - **Matrix Package**: [{package}](https://github.com/{REPO}/tree/{HEAD_BRANCH}/pkgs/{package}) - **Component**: [{comp_file_url}](https://github.com/{REPO}/tree/{HEAD_BRANCH}/{comp_file_url}) - **Test File**: [{test_file_url}](https://github.com/{REPO}/tree/{HEAD_BRANCH}/{test_file_url}) -- **Branch**: [{HEAD_BRANCH}](https://github.com/{REPO}/tree/{HEAD_BRANCH}) => [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) +- **Branches**: [{HEAD_BRANCH}](https://github.com/{REPO}/tree/{HEAD_BRANCH}) ==> [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) - **Changes**: [View Changes](https://github.com/{REPO}/commit/{COMMIT_SHA}) - **Files**: [View Files](https://github.com/{REPO}/tree/{COMMIT_SHA}) - **Workflow Run**: [View Run]({WORKFLOW_RUN_URL}) @@ -181,7 +181,7 @@ def add_comment_to_issue(issue_number, test, package): - **Matrix Package**: [{package}](https://github.com/{REPO}/tree/{HEAD_BRANCH}/pkgs/{package}) - **Component**: [{comp_file_url}](https://github.com/{REPO}/tree/{HEAD_BRANCH}/{comp_file_url}) - **Test File**: [{test_file_url}](https://github.com/{REPO}/tree/{HEAD_BRANCH}/{test_file_url}) -- **Branch**: [{HEAD_BRANCH}](https://github.com/{REPO}/tree/{HEAD_BRANCH}) => [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) +- **Branches**: [{HEAD_BRANCH}](https://github.com/{REPO}/tree/{HEAD_BRANCH}) ==> [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) - **Changes**: [View Changes](https://github.com/{REPO}/commit/{COMMIT_SHA}) - **Files**: [View Files](https://github.com/{REPO}/tree/{COMMIT_SHA}) - **Workflow Run**: [View Run]({WORKFLOW_RUN_URL}) From d45d356e77bb83b4afdcb5304f5c9ce6d8358f08 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Wed, 18 Dec 2024 04:39:11 -0600 Subject: [PATCH 163/168] Update pyproject.toml --- pkgs/community/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/community/pyproject.toml b/pkgs/community/pyproject.toml index d8feb1490..ce1fb5c92 100644 --- a/pkgs/community/pyproject.toml +++ b/pkgs/community/pyproject.toml @@ -78,6 +78,7 @@ pytest-asyncio = ">=0.24.0" pytest-xdist = "^3.6.1" pytest-json-report = "^1.5.0" python-dotenv = "*" +requests = "^2.32.3" [build-system] requires = ["poetry-core>=1.0.0"] From 6ac16f50cb90e6d77eb37bbd76091a9984f89341 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Wed, 18 Dec 2024 05:01:29 -0600 Subject: [PATCH 164/168] comm - Update pyproject.toml --- pkgs/community/pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkgs/community/pyproject.toml b/pkgs/community/pyproject.toml index ce1fb5c92..76584e7d0 100644 --- a/pkgs/community/pyproject.toml +++ b/pkgs/community/pyproject.toml @@ -16,6 +16,10 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.13" captcha = "^0.6.0" + +# We should remove and only rely on httpx +requests = "^2.32.3" + chromadb = { version = "^0.5.17", optional = true } duckdb = { version = "^1.1.1", optional = true } folium = { version = "^0.18.0", optional = true } From c2a4906fc5959d57ba4104da313de812814f2e49 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 19 Dec 2024 17:08:09 -0600 Subject: [PATCH 165/168] Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index 8cdc38823..42e6ae449 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -145,7 +145,8 @@ jobs: poetry run pytest ${{ matrix.package_tests.tests }} --tb=short --json-report --json-report-file=pytest_results.json || true - name: Process test results and manage issues - if: always() + if: | + github.event.pull_request.head.repo.full_name == github.repository && always() env: DEEPINFRA_API_KEY: ${{ secrets.DEEPINFRA_API_KEY }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From f9946f3853ed11bd3f5e4029827b1bbc77a93871 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 19 Dec 2024 17:55:49 -0600 Subject: [PATCH 166/168] Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index 42e6ae449..af887d407 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -142,7 +142,17 @@ jobs: echo "Running tests for package: ${{ matrix.package_tests.package }}" echo "Test files: ${{ matrix.package_tests.tests }}" cd pkgs/${{ matrix.package_tests.package }} - poetry run pytest ${{ matrix.package_tests.tests }} --tb=short --json-report --json-report-file=pytest_results.json || true + poetry run pytest -v ${{ matrix.package_tests.tests }} --junitxml=results.xml -n 4 --dist=loadfile --tb=short --json-report --json-report-file=pytest_results.json || true + + - name: Output test results for debugging + run: | + ls + cat pkgs/${{ matrix.package }}/results.xml + + - name: Classify test results + run: | + python scripts/classify_results.py pkgs/${{ matrix.package_tests.package }}/results.xml + continue-on-error: false - name: Process test results and manage issues if: | From 0c8a5c0a7660ba6c5a93d12ec5211eb832f9f5c9 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 19 Dec 2024 18:00:15 -0600 Subject: [PATCH 167/168] Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index af887d407..f7ffa1ab2 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -146,8 +146,8 @@ jobs: - name: Output test results for debugging run: | - ls - cat pkgs/${{ matrix.package }}/results.xml + ls -R + cat pkgs/${{ matrix.package_tests.package }}/results.xml - name: Classify test results run: | @@ -164,5 +164,4 @@ jobs: run: | cd pkgs/swarmauri # Change to the directory containing pyproject.toml poetry run python ../../scripts/rag_issue_manager.py --results-file=pytest_results.json --package=${{ matrix.package_tests.package }} - - + From 84404c2708bba654cd94533f1ef359c002c1ba8c Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Thu, 19 Dec 2024 18:13:18 -0600 Subject: [PATCH 168/168] Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index f7ffa1ab2..f9935e520 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -142,17 +142,18 @@ jobs: echo "Running tests for package: ${{ matrix.package_tests.package }}" echo "Test files: ${{ matrix.package_tests.tests }}" cd pkgs/${{ matrix.package_tests.package }} - poetry run pytest -v ${{ matrix.package_tests.tests }} --junitxml=results.xml -n 4 --dist=loadfile --tb=short --json-report --json-report-file=pytest_results.json || true + poetry run pytest -v ${{ matrix.package_tests.tests }} -n 4 --dist=loadfile --tb=short --json-report --json-report-file=pytest_results.json || true + - name: Output test results for debugging run: | ls -R - cat pkgs/${{ matrix.package_tests.package }}/results.xml + cat pkgs/${{ matrix.package_tests.package }}/pytest_results.json - - name: Classify test results - run: | - python scripts/classify_results.py pkgs/${{ matrix.package_tests.package }}/results.xml - continue-on-error: false + # - name: Classify test results + # run: | + # python scripts/classify_results.py pkgs/${{ matrix.package_tests.package }}/results.xml + # continue-on-error: false - name: Process test results and manage issues if: |