From 380d0f57e02767c39e857d9da291bbb7dd41accb Mon Sep 17 00:00:00 2001 From: 3rdSon Date: Sat, 30 Nov 2024 11:11:07 +0100 Subject: [PATCH 01/95] swarm -googledataconnector --- .../concrete/GoogleDriveDataConnector.py | 341 ++++++++++++++++++ .../dataconnectors/concrete/__init__.py | 16 + pkgs/swarmauri/tests/static/credentials.json | 5 + .../GoogleDriveDataConnector_unit_test.py | 86 +++++ 4 files changed, 448 insertions(+) create mode 100644 pkgs/swarmauri/swarmauri/dataconnectors/concrete/GoogleDriveDataConnector.py create mode 100644 pkgs/swarmauri/swarmauri/dataconnectors/concrete/__init__.py create mode 100644 pkgs/swarmauri/tests/static/credentials.json create mode 100644 pkgs/swarmauri/tests/unit/dataconnectors/GoogleDriveDataConnector_unit_test.py diff --git a/pkgs/swarmauri/swarmauri/dataconnectors/concrete/GoogleDriveDataConnector.py b/pkgs/swarmauri/swarmauri/dataconnectors/concrete/GoogleDriveDataConnector.py new file mode 100644 index 000000000..57fa66a2a --- /dev/null +++ b/pkgs/swarmauri/swarmauri/dataconnectors/concrete/GoogleDriveDataConnector.py @@ -0,0 +1,341 @@ +import logging +from urllib.parse import urlencode +import httpx +import base64 +import json +from typing import List +from swarmauri.documents.base.DocumentBase import DocumentBase +from swarmauri.dataconnectors.base.DataConnectorBase import DataConnectorBase + + +class GoogleDriveDataConnector(DataConnectorBase): + """ + Data connector for interacting with Google Drive files and converting them to Swarmauri documents. + + Supports authentication, data fetching, and basic CRUD operations for Google Drive resources. + """ + + def __init__(self, credentials_path: str = None): + """ + Initialize the Google Drive Data Connector. + + :param credentials_path: Path to the Google OAuth2 credentials JSON file + """ + with open(credentials_path, "r") as cred_file: + credentials = json.load(cred_file) + + self.client_id = credentials.get("client_id") + self.client_secret = credentials.get("client_secret") + self.redirect_uri = credentials.get("redirect_uri") + + # Tokens will be stored here + self.access_token = None + self.refresh_token = None + + self.authorization_code = None + + self.client = httpx.Client() + + def generate_authorization_url(self) -> str: + """Generate the authorization URL for user consent""" + params = { + "client_id": self.client_id, + "redirect_uri": self.redirect_uri, + "response_type": "code", + "scope": "https://www.googleapis.com/auth/drive", + "access_type": "offline", # This ensures we get a refresh token + } + return f"https://accounts.google.com/o/oauth2/v2/auth?{urlencode(params)}" + + def _exchange_code_for_tokens(self): + """Exchange authorization code for access and refresh tokens""" + if not self.authorization_code: + raise ValueError("No authorization code available") + + token_url = "https://oauth2.googleapis.com/token" + payload = { + "client_id": self.client_id, + "client_secret": self.client_secret, + "code": self.authorization_code, + "grant_type": "authorization_code", + "redirect_uri": self.redirect_uri, + } + + response = self.client.post(token_url, data=payload) + tokens = response.json() + + logging.info(f"Token response: {tokens}") + if "access_token" not in tokens: + raise ValueError("Failed to obtain access token") + self.access_token = tokens["access_token"] + self.refresh_token = tokens.get("refresh_token") + + def refresh_access_token(self): + """Refresh the access token using the refresh token""" + if not self.refresh_token: + raise ValueError("No refresh token available") + + token_url = "https://oauth2.googleapis.com/token" + payload = { + "client_id": self.client_id, + "client_secret": self.client_secret, + "refresh_token": self.refresh_token, + "grant_type": "refresh_token", + } + + response = self.client.post(token_url, data=payload) + tokens = response.json() + self.access_token = tokens["access_token"] + + def authenticate(self): + """ + Authenticate with Google Drive using OAuth2. + + This method generates an authorization URL, prompts the user to visit the URL + and enter the authorization code, and then exchanges the code for tokens. + """ + try: + # Generate authorization URL + auth_url = self.generate_authorization_url() + print("Please visit the following URL to authenticate:") + print(auth_url) + + # Prompt for authorization code + while True: + authorization_code = input("Enter the authorization code: ").strip() + + if not authorization_code: + print("Authorization code cannot be empty. Please try again.") + continue + + self.authorization_code = authorization_code + + try: + self._exchange_code_for_tokens() + logging.info("Successfully authenticated and obtained tokens") + return + except ValueError as e: + print(f"Error exchanging authorization code: {e}") + print("Please try again.") + self.authorization_code = None + + except Exception as e: + logging.error(f"Authentication failed: {e}") + raise ValueError(f"Authentication failed: {e}") + + def fetch_data(self, query: str = None, **kwargs) -> List[DocumentBase]: + """ + Fetch documents from Google Drive based on a query. + + :param query: Search query for files (optional) + :param kwargs: Additional parameters like mime_type, max_results + :return: List of Swarmauri Documents + """ + if not self.access_token: + raise ValueError("Not authenticated. Call authenticate() first.") + + try: + # Prepare request parameters + query_str = query or "" + mime_type = kwargs.get("mime_type", "application/vnd.google-apps.document") + max_results = kwargs.get("max_results", 100) + + # Construct request headers and parameters + headers = { + "Authorization": f"Bearer {self.access_token}", + "Accept": "application/json", + } + + params = { + "q": f"mimeType='{mime_type}' and name contains '{query_str}'", + "pageSize": max_results, + "fields": "files(id,name,mimeType)", + } + + # Make request to Google Drive API + response = self.client.get( + "https://www.googleapis.com/drive/v3/files", + headers=headers, + params=params, + ) + response.raise_for_status() + + files = response.json().get("files", []) + + # Convert Google Drive files to Swarmauri Documents + documents = [] + for file in files: + content = self._get_file_content(file["id"]) + document = DocumentBase( + content=content, + metadata={ + "id": file["id"], + "name": file["name"], + "mime_type": file["mimeType"], + }, + ) + documents.append(document) + + return documents + + except httpx.HTTPError as error: + raise ValueError(f"Error fetching Google Drive files: {error}") + + def _get_file_content(self, file_id: str) -> str: + """ + Retrieve text content from a Google Drive file. + + :param file_id: ID of the Google Drive file + :return: Text content of the file + """ + try: + # Prepare export request + headers = {"Authorization": f"Bearer {self.access_token}"} + + # Export file as plain text + export_url = f"https://www.googleapis.com/drive/v3/files/{file_id}/export" + params = {"mimeType": "text/plain"} + + response = self.client.get(export_url, headers=headers, params=params) + response.raise_for_status() + + return response.text + + except httpx.HTTPError as error: + print(f"An error occurred retrieving file content: {error}") + return "" + + def insert_data(self, data, **kwargs): + """ + Insert a new file into Google Drive. + + :param data: Content of the file to be inserted + :param kwargs: Additional metadata like filename, mime_type + :return: ID of the inserted file + """ + if not self.access_token: + raise ValueError("Not authenticated. Call authenticate() first.") + + try: + headers = { + "Authorization": f"Bearer {self.access_token}", + "Content-Type": "application/json", + } + + # Prepare file metadata + file_metadata = { + "name": kwargs.get("filename", "Untitled Document"), + "mimeType": kwargs.get( + "mime_type", "application/vnd.google-apps.document" + ), + } + + # Prepare file content (base64 encoded) + media_content = base64.b64encode(data.encode("utf-8")).decode("utf-8") + + # Construct payload + payload = { + "metadata": file_metadata, + "media": {"mimeType": "text/plain", "body": media_content}, + } + + # Make request to create file + response = self.client.post( + "https://www.googleapis.com/upload/drive/v3/files", + headers=headers, + json=payload, + ) + response.raise_for_status() + + return response.json().get("id") + + except httpx.HTTPError as error: + raise ValueError(f"Error inserting file: {error}") + + def update_data(self, identifier, data, **kwargs): + """ + Update an existing Google Drive file. + + :param identifier: File ID to update + :param data: New content for the file + :param kwargs: Additional update parameters + """ + if not self.access_token: + raise ValueError("Not authenticated. Call authenticate() first.") + + try: + headers = { + "Authorization": f"Bearer {self.access_token}", + "Content-Type": "application/json", + } + + # Prepare file content (base64 encoded) + media_content = base64.b64encode(data.encode("utf-8")).decode("utf-8") + + # Construct payload + payload = {"media": {"mimeType": "text/plain", "body": media_content}} + + # Make request to update file + response = self.client.patch( + f"https://www.googleapis.com/upload/drive/v3/files/{identifier}", + headers=headers, + json=payload, + ) + response.raise_for_status() + + except httpx.HTTPError as error: + raise ValueError(f"Error updating file: {error}") + + def delete_data(self, identifier, **kwargs): + """ + Delete a file from Google Drive. + + :param identifier: File ID to delete + """ + if not self.access_token: + raise ValueError("Not authenticated. Call authenticate() first.") + + try: + headers = {"Authorization": f"Bearer {self.access_token}"} + + response = self.client.delete( + f"https://www.googleapis.com/drive/v3/files/{identifier}", + headers=headers, + ) + response.raise_for_status() + + except httpx.HTTPError as error: + raise ValueError(f"Error deleting file: {error}") + + def test_connection(self, **kwargs): + """ + Test the connection to Google Drive by listing files. + + :return: Boolean indicating connection success + """ + try: + if not self.access_token: + self.authenticate(**kwargs) + + # Prepare request headers + headers = { + "Authorization": f"Bearer {self.access_token}", + "Accept": "application/json", + } + + # List first 5 files to test connection + params = {"pageSize": 5, "fields": "files(id,name)"} + + response = self.client.get( + "https://www.googleapis.com/drive/v3/files", + headers=headers, + params=params, + ) + response.raise_for_status() + + files = response.json().get("files", []) + return len(files) > 0 + + except Exception as e: + print(f"Connection test failed: {e}") + return False diff --git a/pkgs/swarmauri/swarmauri/dataconnectors/concrete/__init__.py b/pkgs/swarmauri/swarmauri/dataconnectors/concrete/__init__.py new file mode 100644 index 000000000..612d7f2e4 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/dataconnectors/concrete/__init__.py @@ -0,0 +1,16 @@ +from swarmauri.utils._lazy_import import _lazy_import + +# List of data connector files names (file names without the ".py" extension) and corresponding class names +data_connector_files = [ + ( + "swarmauri.dataconnectors.concrete.GoogleDriveDataConnector", + "GoogleDriveDataConnector", + ), +] + +# Lazy loading of data connector classes, storing them in variables +for module_name, class_name in data_connector_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded data connector classes to __all__ +__all__ = [class_name for _, class_name in data_connector_files] diff --git a/pkgs/swarmauri/tests/static/credentials.json b/pkgs/swarmauri/tests/static/credentials.json new file mode 100644 index 000000000..a36c5e36b --- /dev/null +++ b/pkgs/swarmauri/tests/static/credentials.json @@ -0,0 +1,5 @@ +{ + "client_id": "Put your client_id here", + "client_secret": "Put your client_secret here", + "redirect_uri": "put your redirect_uri here or use :ietf:wg:oauth:2.0:oob" +} \ No newline at end of file diff --git a/pkgs/swarmauri/tests/unit/dataconnectors/GoogleDriveDataConnector_unit_test.py b/pkgs/swarmauri/tests/unit/dataconnectors/GoogleDriveDataConnector_unit_test.py new file mode 100644 index 000000000..9b1a092da --- /dev/null +++ b/pkgs/swarmauri/tests/unit/dataconnectors/GoogleDriveDataConnector_unit_test.py @@ -0,0 +1,86 @@ +import pytest +from swarmauri.dataconnectors.concrete.GoogleDriveDataConnector import ( + GoogleDriveDataConnector, +) + + +@pytest.fixture(scope="module") +def authenticated_connector(): + """Authenticate the GoogleDriveDataConnector once for the test suite.""" + # Path to the valid credentials JSON file + credentials_path = "pkgs/swarmauri/tests/static/credentials.json" + connector = GoogleDriveDataConnector(credentials_path=credentials_path) + + # Perform authentication once + try: + connector.authenticate() # Requires manual input for the authorization code + except Exception as e: + pytest.fail(f"Authentication failed: {e}") + + return connector + + +@pytest.fixture(scope="module") +def shared_file_id(): + """Return a shared file ID for testing.""" + return {} + + +@pytest.mark.skip(reason="Skipping test_generate_authorization_url") +def test_generate_authorization_url(): + """Test generate_authorization_url without authentication.""" + # Path to the valid credentials JSON file + credentials_path = "pkgs/swarmauri/tests/static/credentials.json" + connector = GoogleDriveDataConnector(credentials_path=credentials_path) + url = connector.generate_authorization_url() + assert isinstance(url, str) + assert "client_id" in url + assert "redirect_uri" in url + assert "https://accounts.google.com/o/oauth2/v2/auth" in url + + +@pytest.mark.skip(reason="Skipping test_fetch_data") +def test_fetch_data(authenticated_connector): + """Test fetching data from Google Drive.""" + documents = authenticated_connector.fetch_data(query="test") + assert isinstance(documents, list) + if documents: + assert all(hasattr(doc, "content") for doc in documents) + assert all(hasattr(doc, "metadata") for doc in documents) + + +@pytest.mark.skip(reason="Skipping test_insert_data") +def test_insert_data(authenticated_connector, shared_file_id): + """Test inserting data into Google Drive.""" + test_data = "Sample content for Google Drive file" + file_id = authenticated_connector.insert_data(test_data, filename="test_file.txt") + assert isinstance(file_id, str) + shared_file_id["file_id"] = file_id + + +@pytest.mark.skip(reason="Skipping test_update_data") +def test_update_data(authenticated_connector, shared_file_id): + """Test updating data in Google Drive.""" + file_id = shared_file_id["file_id"] + updated_content = "Updated content for Google Drive file" + try: + authenticated_connector.update_data(file_id, updated_content) + except Exception as e: + pytest.fail(f"Failed to update file: {e}") + + +@pytest.mark.skip(reason="Skipping test_delete_data") +def test_delete_data(authenticated_connector, shared_file_id): + """Test deleting data from Google Drive.""" + file_id = shared_file_id["file_id"] # Replace with an actual file ID + try: + authenticated_connector.delete_data(file_id) + except Exception as e: + pytest.fail(f"Failed to delete file: {e}") + + +@pytest.mark.skip(reason="Skipping test_connection") +def test_connection(authenticated_connector): + """Test the connection to Google Drive.""" + connection_success = authenticated_connector.test_connection() + assert connection_success is True From ec023dab88b7acdc15960a4814169f5a95ace533 Mon Sep 17 00:00:00 2001 From: 3rdSon Date: Wed, 11 Dec 2024 15:25:39 +0100 Subject: [PATCH 02/95] Base and core transport --- .../swarmauri_core/transport/ITransport.py | 42 ++++++++++++++ .../core/swarmauri_core/transport/__init__.py | 42 ++++++++++++++ .../swarmauri/swarmauri/transport/__init__.py | 1 + .../swarmauri/transport/base/TransportBase.py | 55 +++++++++++++++++++ .../swarmauri/transport/base/__init__.py | 0 .../swarmauri/transport/concrete/__init__.py | 0 6 files changed, 140 insertions(+) create mode 100644 pkgs/core/swarmauri_core/transport/ITransport.py create mode 100644 pkgs/core/swarmauri_core/transport/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/transport/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/transport/base/TransportBase.py create mode 100644 pkgs/swarmauri/swarmauri/transport/base/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/transport/concrete/__init__.py diff --git a/pkgs/core/swarmauri_core/transport/ITransport.py b/pkgs/core/swarmauri_core/transport/ITransport.py new file mode 100644 index 000000000..0d621efbb --- /dev/null +++ b/pkgs/core/swarmauri_core/transport/ITransport.py @@ -0,0 +1,42 @@ +from abc import ABC, abstractmethod +from typing import Any, List + + +class ITransportComm(ABC): + """ + Interface defining standard transportation methods for agent interactions + """ + @abstractmethod + def send(self, sender: str, recipient: str, message: Any) -> None: + """ + Send a message to a specific recipient + """ + pass + + @abstractmethod + def broadcast(self, sender: str, message: Any) -> None: + """ + Broadcast a message to all agents + """ + pass + + @abstractmethod + def multicast(self, sender: str, recipients: List[str], message: Any) -> None: + """ + Send a message to multiple specific recipients + """ + pass + + @abstractmethod + def subscribe(self, topic: str, subscriber: str) -> None: + """ + Subscribe to a specific transportation topic + """ + pass + + @abstractmethod + def publish(self, topic: str, message: Any) -> None: + """ + Publish a message to a specific topic + """ + pass \ No newline at end of file diff --git a/pkgs/core/swarmauri_core/transport/__init__.py b/pkgs/core/swarmauri_core/transport/__init__.py new file mode 100644 index 000000000..4cb714616 --- /dev/null +++ b/pkgs/core/swarmauri_core/transport/__init__.py @@ -0,0 +1,42 @@ +from abc import ABC, abstractmethod +from typing import Any, List + + +class ITransportComm(ABC): + """ + Interface defining standard communication methods for agent interactions + """ + @abstractmethod + def send(self, sender: str, recipient: str, message: Any) -> None: + """ + Send a message to a specific recipient + """ + pass + + @abstractmethod + def broadcast(self, sender: str, message: Any) -> None: + """ + Broadcast a message to all agents + """ + pass + + @abstractmethod + def multicast(self, sender: str, recipients: List[str], message: Any) -> None: + """ + Send a message to multiple specific recipients + """ + pass + + @abstractmethod + def subscribe(self, topic: str, subscriber: str) -> None: + """ + Subscribe to a specific communication topic + """ + pass + + @abstractmethod + def publish(self, topic: str, message: Any) -> None: + """ + Publish a message to a specific topic + """ + pass \ No newline at end of file diff --git a/pkgs/swarmauri/swarmauri/transport/__init__.py b/pkgs/swarmauri/swarmauri/transport/__init__.py new file mode 100644 index 000000000..f3ada0e57 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/transport/__init__.py @@ -0,0 +1 @@ +from swarmauri.transport.concrete import * diff --git a/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py b/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py new file mode 100644 index 000000000..1d6d35781 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py @@ -0,0 +1,55 @@ +from typing import Dict, Any, List +from enum import Enum, auto +import datetime +from swarmauri_core.transport.ITransport import ITransport + +class TransportationProtocol(Enum): + """ + Enumeration of transportation protocols supported by the transport layer + """ + UNICAST = auto() + MULTICAST = auto() + BROADCAST = auto() + PUBSUB = auto() + +class TransportBase(ITransport): + """ + Base class for transport transportation with shared utilities + """ + def __init__(self, name: str): + self.name = name + self.subscriptions: Dict[str, List[str]] = {} + self.message_history: List[Dict[str, Any]] = [] + + def log_message(self, sender: str, recipients: List[str], message: Any, protocol: TransportationProtocol): + """ + Log transportation events + """ + log_entry = { + 'sender': sender, + 'recipients': recipients, + 'message': message, + 'protocol': protocol, + 'timestamp': datetime.now() + } + self.message_history.append(log_entry) + + def send(self, sender: str, recipient: str, message: Any) -> None: + raise NotImplementedError("Subclasses must implement send method") + + def broadcast(self, sender: str, message: Any) -> None: + raise NotImplementedError("Subclasses must implement broadcast method") + + def multicast(self, sender: str, recipients: List[str], message: Any) -> None: + raise NotImplementedError("Subclasses must implement multicast method") + + def subscribe(self, topic: str, subscriber: str) -> None: + if topic not in self.subscriptions: + self.subscriptions[topic] = [] + if subscriber not in self.subscriptions[topic]: + self.subscriptions[topic].append(subscriber) + + def publish(self, topic: str, message: Any) -> None: + if topic in self.subscriptions: + for subscriber in self.subscriptions[topic]: + self.send(topic, subscriber, message) \ No newline at end of file diff --git a/pkgs/swarmauri/swarmauri/transport/base/__init__.py b/pkgs/swarmauri/swarmauri/transport/base/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/transport/concrete/__init__.py b/pkgs/swarmauri/swarmauri/transport/concrete/__init__.py new file mode 100644 index 000000000..e69de29bb From 46f655d85c38b6ff8fb7281ac0f3711b4aae9baf Mon Sep 17 00:00:00 2001 From: 3rdSon Date: Thu, 12 Dec 2024 13:41:39 +0100 Subject: [PATCH 03/95] corrected the base class --- pkgs/core/swarmauri_core/ComponentBase.py | 1 + .../swarmauri_core/transport/ITransport.py | 17 +---- .../core/swarmauri_core/transport/__init__.py | 42 ------------ .../swarmauri/transport/base/TransportBase.py | 65 +++++++++---------- .../transport/concrete/PubSubTransport.py | 0 .../swarmauri/transport/concrete/__init__.py | 13 ++++ .../transport/PubSubTransport_unit_test.py | 0 7 files changed, 45 insertions(+), 93 deletions(-) create mode 100644 pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py create mode 100644 pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py diff --git a/pkgs/core/swarmauri_core/ComponentBase.py b/pkgs/core/swarmauri_core/ComponentBase.py index 13a096001..e6f3a4d5c 100644 --- a/pkgs/core/swarmauri_core/ComponentBase.py +++ b/pkgs/core/swarmauri_core/ComponentBase.py @@ -53,6 +53,7 @@ class ResourceTypes(Enum): VECTOR = "Vector" VCM = "VCM" DATA_CONNECTOR = "DataConnector" + TRANSPORT = "Transport" def generate_id() -> str: diff --git a/pkgs/core/swarmauri_core/transport/ITransport.py b/pkgs/core/swarmauri_core/transport/ITransport.py index 0d621efbb..c9f811473 100644 --- a/pkgs/core/swarmauri_core/transport/ITransport.py +++ b/pkgs/core/swarmauri_core/transport/ITransport.py @@ -2,10 +2,11 @@ from typing import Any, List -class ITransportComm(ABC): +class ITransport(ABC): """ Interface defining standard transportation methods for agent interactions """ + @abstractmethod def send(self, sender: str, recipient: str, message: Any) -> None: """ @@ -26,17 +27,3 @@ def multicast(self, sender: str, recipients: List[str], message: Any) -> None: Send a message to multiple specific recipients """ pass - - @abstractmethod - def subscribe(self, topic: str, subscriber: str) -> None: - """ - Subscribe to a specific transportation topic - """ - pass - - @abstractmethod - def publish(self, topic: str, message: Any) -> None: - """ - Publish a message to a specific topic - """ - pass \ No newline at end of file diff --git a/pkgs/core/swarmauri_core/transport/__init__.py b/pkgs/core/swarmauri_core/transport/__init__.py index 4cb714616..e69de29bb 100644 --- a/pkgs/core/swarmauri_core/transport/__init__.py +++ b/pkgs/core/swarmauri_core/transport/__init__.py @@ -1,42 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Any, List - - -class ITransportComm(ABC): - """ - Interface defining standard communication methods for agent interactions - """ - @abstractmethod - def send(self, sender: str, recipient: str, message: Any) -> None: - """ - Send a message to a specific recipient - """ - pass - - @abstractmethod - def broadcast(self, sender: str, message: Any) -> None: - """ - Broadcast a message to all agents - """ - pass - - @abstractmethod - def multicast(self, sender: str, recipients: List[str], message: Any) -> None: - """ - Send a message to multiple specific recipients - """ - pass - - @abstractmethod - def subscribe(self, topic: str, subscriber: str) -> None: - """ - Subscribe to a specific communication topic - """ - pass - - @abstractmethod - def publish(self, topic: str, message: Any) -> None: - """ - Publish a message to a specific topic - """ - pass \ No newline at end of file diff --git a/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py b/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py index 1d6d35781..2020f677f 100644 --- a/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py +++ b/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py @@ -1,55 +1,48 @@ -from typing import Dict, Any, List +from typing import Dict, Any, List, Optional, Literal from enum import Enum, auto -import datetime +from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes from swarmauri_core.transport.ITransport import ITransport + class TransportationProtocol(Enum): """ Enumeration of transportation protocols supported by the transport layer """ + UNICAST = auto() MULTICAST = auto() BROADCAST = auto() PUBSUB = auto() -class TransportBase(ITransport): - """ - Base class for transport transportation with shared utilities - """ - def __init__(self, name: str): - self.name = name - self.subscriptions: Dict[str, List[str]] = {} - self.message_history: List[Dict[str, Any]] = [] - def log_message(self, sender: str, recipients: List[str], message: Any, protocol: TransportationProtocol): - """ - Log transportation events - """ - log_entry = { - 'sender': sender, - 'recipients': recipients, - 'message': message, - 'protocol': protocol, - 'timestamp': datetime.now() - } - self.message_history.append(log_entry) +class TransportBase(ITransport, ComponentBase): + allowed_protocols: List[TransportationProtocol] = [] + resource: Optional[str] = ResourceTypes.TRANSPORT.value + type: Literal["TransportBase"] = "TransportBase" def send(self, sender: str, recipient: str, message: Any) -> None: - raise NotImplementedError("Subclasses must implement send method") + """ + Send a message to a specific recipient. + + Raises: + NotImplementedError: Subclasses must implement this method. + """ + raise NotImplementedError("send() not implemented in subclass yet.") def broadcast(self, sender: str, message: Any) -> None: - raise NotImplementedError("Subclasses must implement broadcast method") + """ + Broadcast a message to all potential recipients. + + Raises: + NotImplementedError: Subclasses must implement this method. + """ + raise NotImplementedError("broadcast() not implemented in subclass yet.") def multicast(self, sender: str, recipients: List[str], message: Any) -> None: - raise NotImplementedError("Subclasses must implement multicast method") - - def subscribe(self, topic: str, subscriber: str) -> None: - if topic not in self.subscriptions: - self.subscriptions[topic] = [] - if subscriber not in self.subscriptions[topic]: - self.subscriptions[topic].append(subscriber) - - def publish(self, topic: str, message: Any) -> None: - if topic in self.subscriptions: - for subscriber in self.subscriptions[topic]: - self.send(topic, subscriber, message) \ No newline at end of file + """ + Send a message to multiple specific recipients. + + Raises: + NotImplementedError: Subclasses must implement this method. + """ + raise NotImplementedError("multicast() not implemented in subclass yet.") diff --git a/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py b/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/transport/concrete/__init__.py b/pkgs/swarmauri/swarmauri/transport/concrete/__init__.py index e69de29bb..893b547b4 100644 --- a/pkgs/swarmauri/swarmauri/transport/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/transport/concrete/__init__.py @@ -0,0 +1,13 @@ +from swarmauri.utils._lazy_import import _lazy_import + +# List of transport names (file names without the ".py" extension) and corresponding class names +transport_files = [ + ("swarmauri.transport.concrete.PubSubTransport", "PubSubTransport"), +] + +# Lazy loading of transport classes, storing them in variables +for module_name, class_name in transport_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded transport classes to __all__ +__all__ = [class_name for _, class_name in transport_files] diff --git a/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py b/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py new file mode 100644 index 000000000..e69de29bb From e46db2df86e8b09118b581994222eedac3beac3d Mon Sep 17 00:00:00 2001 From: 3rdSon Date: Thu, 12 Dec 2024 14:24:26 +0100 Subject: [PATCH 04/95] corrected the merge issue --- pkgs/core/swarmauri_core/ComponentBase.py | 70 ++++++++++++++++++++++- 1 file changed, 67 insertions(+), 3 deletions(-) diff --git a/pkgs/core/swarmauri_core/ComponentBase.py b/pkgs/core/swarmauri_core/ComponentBase.py index e6f3a4d5c..63b3f6ea8 100644 --- a/pkgs/core/swarmauri_core/ComponentBase.py +++ b/pkgs/core/swarmauri_core/ComponentBase.py @@ -1,12 +1,13 @@ +import json from typing import ( + Any, + Dict, Optional, List, Literal, TypeVar, Type, Union, - Annotated, - Generic, ClassVar, Set, get_args, @@ -16,11 +17,14 @@ from enum import Enum import inspect import hashlib -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field, ValidationError, field_validator import logging from swarmauri_core.typing import SubclassUnion +T = TypeVar("T", bound="ComponentBase") + + class ResourceTypes(Enum): UNIVERSAL_BASE = "ComponentBase" AGENT = "Agent" @@ -54,6 +58,7 @@ class ResourceTypes(Enum): VCM = "VCM" DATA_CONNECTOR = "DataConnector" TRANSPORT = "Transport" + FACTORY = "Factory" def generate_id() -> str: @@ -181,3 +186,62 @@ def swm_path(self): @property def swm_isremote(self): return bool(self.host) + + @classmethod + def model_validate_json( + cls: Type[T], json_payload: Union[str, Dict[str, Any]], strict: bool = False + ) -> T: + # Ensure we're working with a dictionary + if isinstance(json_payload, str): + try: + payload_dict = json.loads(json_payload) + except json.JSONDecodeError: + raise ValueError("Invalid JSON payload") + else: + payload_dict = json_payload + + # Try to determine the specific component type + component_type = payload_dict.get("type", "ComponentBase") + + # Attempt to find the correct subclass + target_cls = cls.get_subclass_by_type(component_type) + + # Fallback logic + if target_cls is None: + if strict: + raise ValueError(f"Cannot resolve component type: {component_type}") + target_cls = cls + logging.warning( + f"Falling back to base ComponentBase for type: {component_type}" + ) + + # Validate using the determined class + try: + return target_cls.model_validate(payload_dict) + except ValidationError as e: + logging.error(f"Validation failed for {component_type}: {e}") + raise + + @classmethod + def get_subclass_by_type(cls, type_name: str) -> Optional[Type["ComponentBase"]]: + # First, check for exact match in registered subclasses + for subclass in cls.__swm_subclasses__: + if ( + subclass.__name__ == type_name + or getattr(subclass, "type", None) == type_name + ): + return subclass + + # If no exact match, try case-insensitive search + for subclass in cls.__swm_subclasses__: + if ( + subclass.__name__.lower() == type_name.lower() + or str(getattr(subclass, "type", "")).lower() == type_name.lower() + ): + return subclass + + return None + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "ComponentBase": + return cls.model_validate(data) From 04ec1427ff997b6c4a042839f52d286d4f950e4b Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Thu, 12 Dec 2024 14:40:47 +0100 Subject: [PATCH 05/95] swarm - feat: added service registry --- pkgs/core/swarmauri_core/ComponentBase.py | 1 + .../service_registries/IServiceRegistry.py | 29 ++++++++++++ .../service_registries/__init__.py | 0 .../swarmauri/service_registries/__init__.py | 0 .../base/ServiceRegistryBase.py | 40 ++++++++++++++++ .../service_registries/base/__init__.py | 0 .../concrete/ServiceRegistry.py | 10 ++++ .../service_registries/concrete/__init__.py | 0 .../ServiceRegistry_unit_test.py | 46 +++++++++++++++++++ 9 files changed, 126 insertions(+) create mode 100644 pkgs/core/swarmauri_core/service_registries/IServiceRegistry.py create mode 100644 pkgs/core/swarmauri_core/service_registries/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/service_registries/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/service_registries/base/ServiceRegistryBase.py create mode 100644 pkgs/swarmauri/swarmauri/service_registries/base/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/service_registries/concrete/ServiceRegistry.py create mode 100644 pkgs/swarmauri/swarmauri/service_registries/concrete/__init__.py create mode 100644 pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py diff --git a/pkgs/core/swarmauri_core/ComponentBase.py b/pkgs/core/swarmauri_core/ComponentBase.py index a5e558380..4aad0dd25 100644 --- a/pkgs/core/swarmauri_core/ComponentBase.py +++ b/pkgs/core/swarmauri_core/ComponentBase.py @@ -58,6 +58,7 @@ class ResourceTypes(Enum): VCM = "VCM" DATA_CONNECTOR = "DataConnector" FACTORY = "Factory" + SERVICE_REGISTRY = "ServiceRegistry" def generate_id() -> str: diff --git a/pkgs/core/swarmauri_core/service_registries/IServiceRegistry.py b/pkgs/core/swarmauri_core/service_registries/IServiceRegistry.py new file mode 100644 index 000000000..e434514aa --- /dev/null +++ b/pkgs/core/swarmauri_core/service_registries/IServiceRegistry.py @@ -0,0 +1,29 @@ +from abc import ABC, abstractmethod +from typing import Dict, Any, List, Optional + + +class IServiceRegistry(ABC): + """ + Abstract base class for service registries. + """ + + @abstractmethod + def register_service(self, name: str, details: Dict[str, Any]) -> None: + """ + Register a new service with the given name and details. + """ + pass + + @abstractmethod + def get_service(self, name: str) -> Optional[Dict[str, Any]]: + """ + Retrieve a service by its name. + """ + pass + + @abstractmethod + def get_services_by_roles(self, roles: List[str]) -> List[str]: + """ + Get services filtered by their roles. + """ + pass diff --git a/pkgs/core/swarmauri_core/service_registries/__init__.py b/pkgs/core/swarmauri_core/service_registries/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/service_registries/__init__.py b/pkgs/swarmauri/swarmauri/service_registries/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/service_registries/base/ServiceRegistryBase.py b/pkgs/swarmauri/swarmauri/service_registries/base/ServiceRegistryBase.py new file mode 100644 index 000000000..ca6a8824d --- /dev/null +++ b/pkgs/swarmauri/swarmauri/service_registries/base/ServiceRegistryBase.py @@ -0,0 +1,40 @@ +from typing import Dict, Any, List, Literal, Optional + +from pydantic import ConfigDict, Field +from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes +from swarmauri_core.service_registries.IServiceRegistry import IServiceRegistry + + +class ServiceRegistryBase(IServiceRegistry, ComponentBase): + """ + Concrete implementation of the IServiceRegistry abstract base class. + """ + + services: Dict[str, Any] = {} + type: Literal["ServiceRegistryBase"] = "ServiceRegistryBase" + resource: Optional[str] = Field( + default=ResourceTypes.SERVICE_REGISTRY.value, frozen=True + ) + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + + def register_service(self, name: str, details: Dict[str, Any]) -> None: + """ + Register a new service with the given name and details. + """ + self.services[name] = details + + def get_service(self, name: str) -> Optional[Dict[str, Any]]: + """ + Retrieve a service by its name. + """ + return self.services.get(name) + + def get_services_by_roles(self, roles: List[str]) -> List[str]: + """ + Get services filtered by their roles. + """ + return [ + name + for name, details in self.services.items() + if details.get("role") in roles + ] diff --git a/pkgs/swarmauri/swarmauri/service_registries/base/__init__.py b/pkgs/swarmauri/swarmauri/service_registries/base/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/service_registries/concrete/ServiceRegistry.py b/pkgs/swarmauri/swarmauri/service_registries/concrete/ServiceRegistry.py new file mode 100644 index 000000000..d61d34452 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/service_registries/concrete/ServiceRegistry.py @@ -0,0 +1,10 @@ +from typing import Literal +from swarmauri.service_registries.base.ServiceRegistryBase import ServiceRegistryBase + + +class ServiceRegistry(ServiceRegistryBase): + """ + Concrete implementation of the ServiceRegistryBase. + """ + + type: Literal["ServiceRegistry"] = "ServiceRegistry" diff --git a/pkgs/swarmauri/swarmauri/service_registries/concrete/__init__.py b/pkgs/swarmauri/swarmauri/service_registries/concrete/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py b/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py new file mode 100644 index 000000000..fca37383a --- /dev/null +++ b/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py @@ -0,0 +1,46 @@ +import pytest +from swarmauri.service_registries.concrete.ServiceRegistry import ServiceRegistry + + +@pytest.fixture +def service_registry(): + return ServiceRegistry() + +@pytest.mark.unit +def test_ubc_resource(service_registry): + assert service_registry.resource == "ServiceRegistry" + + +@pytest.mark.unit +def test_ubc_type(service_registry): + assert service_registry.type == "ServiceRegistry" + + +@pytest.mark.unit +def test_serialization(service_registry): + assert service_registry.id == service_registry.model_validate_json(service_registry.model_dump_json()).id + + +def test_register_service(service_registry): + service_registry.register_service("auth", {"role": "authentication"}) + assert service_registry.services["auth"] == {"role": "authentication"} + + +def test_get_service(service_registry): + service_registry.register_service("auth", {"role": "authentication"}) + service = service_registry.get_service("auth") + assert service == {"role": "authentication"} + assert service_registry.get_service("nonexistent") is None + + +def test_get_services_by_roles(service_registry): + service_registry.register_service("auth", {"role": "authentication"}) + service_registry.register_service("db", {"role": "database"}) + recipients = service_registry.get_services_by_roles(["authentication"]) + assert recipients == ["auth"] + recipients = service_registry.get_services_by_roles(["database"]) + assert recipients == ["db"] + recipients = service_registry.get_services_by_roles( + ["authentication", "database"] + ) + assert set(recipients) == {"auth", "db"} From f031faa5b10ebbead7ca00f21b4d33b2f338ead2 Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Thu, 12 Dec 2024 14:46:28 +0100 Subject: [PATCH 06/95] test: improve formatting and add unit tests for ServiceRegistry --- .../service_registries/ServiceRegistry_unit_test.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py b/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py index fca37383a..9c1622a9d 100644 --- a/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py +++ b/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py @@ -6,6 +6,7 @@ def service_registry(): return ServiceRegistry() + @pytest.mark.unit def test_ubc_resource(service_registry): assert service_registry.resource == "ServiceRegistry" @@ -18,14 +19,19 @@ def test_ubc_type(service_registry): @pytest.mark.unit def test_serialization(service_registry): - assert service_registry.id == service_registry.model_validate_json(service_registry.model_dump_json()).id + assert ( + service_registry.id + == service_registry.model_validate_json(service_registry.model_dump_json()).id + ) +@pytest.mark.unit def test_register_service(service_registry): service_registry.register_service("auth", {"role": "authentication"}) assert service_registry.services["auth"] == {"role": "authentication"} +@pytest.mark.unit def test_get_service(service_registry): service_registry.register_service("auth", {"role": "authentication"}) service = service_registry.get_service("auth") @@ -33,6 +39,7 @@ def test_get_service(service_registry): assert service_registry.get_service("nonexistent") is None +@pytest.mark.unit def test_get_services_by_roles(service_registry): service_registry.register_service("auth", {"role": "authentication"}) service_registry.register_service("db", {"role": "database"}) @@ -40,7 +47,5 @@ def test_get_services_by_roles(service_registry): assert recipients == ["auth"] recipients = service_registry.get_services_by_roles(["database"]) assert recipients == ["db"] - recipients = service_registry.get_services_by_roles( - ["authentication", "database"] - ) + recipients = service_registry.get_services_by_roles(["authentication", "database"]) assert set(recipients) == {"auth", "db"} From e90a8576dde1d2b5f985c041dce3d35cc4c2fe50 Mon Sep 17 00:00:00 2001 From: 3rdSon Date: Thu, 12 Dec 2024 15:47:57 +0100 Subject: [PATCH 07/95] implemented pubsubtransport --- .../transport/concrete/PubSubTransport.py | 122 ++++++++++++++++++ .../transport/PubSubTransport_unit_test.py | 116 +++++++++++++++++ 2 files changed, 238 insertions(+) diff --git a/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py b/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py index e69de29bb..ccc07c221 100644 --- a/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py +++ b/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py @@ -0,0 +1,122 @@ +from typing import Dict, Any, List, Optional, Set +from uuid import uuid4 +import asyncio +from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes +from swarmauri_core.transport.ITransport import ITransport +from swarmauri.transport.base.TransportBase import TransportBase, TransportationProtocol + + +class PubSubTransport(TransportBase): + allowed_protocols: List[TransportationProtocol] = [TransportationProtocol.PUBSUB] + + def __init__(self): + """ + Initialize the Publish-Subscribe Broker. + Manages topics and subscriptions for agents. + """ + super().__init__() + self._topics: Dict[str, Set[str]] = {} # Topic to subscriber mappings + self._subscribers: Dict[str, asyncio.Queue] = ( + {} + ) # Subscriber ID to message queue + + async def subscribe(self, topic: str) -> str: + """ + Subscribe an agent to a specific topic. + + Args: + topic (str): The topic to subscribe to + + Returns: + str: Unique subscriber ID + """ + subscriber_id = str(uuid4()) + + # Create message queue for this subscriber + self._subscribers[subscriber_id] = asyncio.Queue() + + # Add subscriber to topic + if topic not in self._topics: + self._topics[topic] = set() + self._topics[topic].add(subscriber_id) + + return subscriber_id + + async def unsubscribe(self, topic: str, subscriber_id: str): + """ + Unsubscribe an agent from a topic. + + Args: + topic (str): The topic to unsubscribe from + subscriber_id (str): Unique identifier of the subscriber + """ + if topic in self._topics and subscriber_id in self._topics[topic]: + self._topics[topic].remove(subscriber_id) + + # Optional: Clean up if no subscribers remain + if not self._topics[topic]: + del self._topics[topic] + + async def publish(self, topic: str, message: Any): + """ + Publish a message to a specific topic. + + Args: + topic (str): The topic to publish to + message (Any): The message to be published + """ + if topic not in self._topics: + return + + # Distribute message to all subscribers of this topic + for subscriber_id in self._topics[topic]: + await self._subscribers[subscriber_id].put(message) + + async def receive(self, subscriber_id: str) -> Any: + """ + Receive messages for a specific subscriber. + + Args: + subscriber_id (str): Unique identifier of the subscriber + + Returns: + Any: Received message + """ + return await self._subscribers[subscriber_id].get() + + def send(self, sender: str, recipient: str, message: Any) -> None: + """ + Simulate sending a direct message (not applicable in Pub/Sub context). + + Args: + sender (str): The sender ID + recipient (str): The recipient ID + message (Any): The message to send + + Raises: + NotImplementedError: This method is not applicable for Pub/Sub. + """ + raise NotImplementedError("Direct send not supported in Pub/Sub model.") + + def broadcast(self, sender: str, message: Any) -> None: + """ + Broadcast a message to all subscribers of all topics. + + Args: + sender (str): The sender ID + message (Any): The message to broadcast + """ + for topic in self._topics: + asyncio.create_task(self.publish(topic, message)) + + def multicast(self, sender: str, recipients: List[str], message: Any) -> None: + """ + Send a message to specific topics (acting as recipients). + + Args: + sender (str): The sender ID + recipients (List[str]): Topics to send the message to + message (Any): The message to send + """ + for topic in recipients: + asyncio.create_task(self.publish(topic, message)) diff --git a/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py b/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py index e69de29bb..2d2c68f76 100644 --- a/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py +++ b/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py @@ -0,0 +1,116 @@ +import pytest +import asyncio +from uuid import UUID +from typing import Any +from swarmauri.transport.concrete.PubSubTransport import ( + PubSubTransport, +) +from swarmauri.utils.timeout_wrapper import timeout + + +@pytest.fixture +async def pubsub_transport(): + transport = PubSubTransport() + yield transport + + +@timeout(5) +@pytest.mark.unit +@pytest.mark.asyncio +async def test_subscribe(pubsub_transport): + topic = "test_topic" + subscriber_id = await pubsub_transport.subscribe(topic) + + # Validate subscriber ID format + assert isinstance(UUID(subscriber_id), UUID) + + # Ensure subscriber is added to the topic + assert subscriber_id in pubsub_transport._topics[topic] + + +@timeout(5) +@pytest.mark.unit +@pytest.mark.asyncio +async def test_unsubscribe(pubsub_transport): + topic = "test_topic" + subscriber_id = await pubsub_transport.subscribe(topic) + + await pubsub_transport.unsubscribe(topic, subscriber_id) + + # Ensure subscriber is removed from the topic + assert subscriber_id not in pubsub_transport._topics.get(topic, set()) + + +@timeout(5) +@pytest.mark.unit +@pytest.mark.asyncio +async def test_publish_and_receive(pubsub_transport): + topic = "test_topic" + subscriber_id = await pubsub_transport.subscribe(topic) + + message = "Hello, PubSub!" + await pubsub_transport.publish(topic, message) + + # Ensure the subscriber receives the message + received_message = await pubsub_transport.receive(subscriber_id) + assert received_message == message + + +@timeout(5) +@pytest.mark.unit +@pytest.mark.asyncio +async def test_broadcast(pubsub_transport): + topic1 = "topic1" + topic2 = "topic2" + subscriber_id1 = await pubsub_transport.subscribe(topic1) + subscriber_id2 = await pubsub_transport.subscribe(topic2) + + message = "Broadcast Message" + pubsub_transport.broadcast("sender_id", message) + + # Ensure both subscribers receive the message + received_message1 = await pubsub_transport.receive(subscriber_id1) + received_message2 = await pubsub_transport.receive(subscriber_id2) + assert received_message1 == message + assert received_message2 == message + + +@timeout(5) +@pytest.mark.unit +@pytest.mark.asyncio +async def test_multicast(pubsub_transport): + topic1 = "topic1" + topic2 = "topic2" + topic3 = "topic3" + subscriber_id1 = await pubsub_transport.subscribe(topic1) + subscriber_id2 = await pubsub_transport.subscribe(topic2) + subscriber_id3 = await pubsub_transport.subscribe(topic3) + + message = "Multicast Message" + pubsub_transport.multicast("sender_id", [topic1, topic2], message) + + # Ensure only subscribers of specified topics receive the message + received_message1 = await pubsub_transport.receive(subscriber_id1) + received_message2 = await pubsub_transport.receive(subscriber_id2) + assert received_message1 == message + assert received_message2 == message + + try: + await asyncio.wait_for(pubsub_transport.receive(subscriber_id3), timeout=1.0) + pytest.fail("Expected no message, but received one.") + except asyncio.TimeoutError: + pass + + +@timeout(5) +@pytest.mark.unit +@pytest.mark.asyncio +async def test_receive_no_messages(pubsub_transport): + topic = "test_topic" + subscriber_id = await pubsub_transport.subscribe(topic) + + try: + await asyncio.wait_for(pubsub_transport.receive(subscriber_id), timeout=1.0) + pytest.fail("Expected no message, but received one.") + except asyncio.TimeoutError: + pass From 8bf840bfdef78ad8422414ba765edc5b8aeaa698 Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Fri, 13 Dec 2024 08:38:11 +0100 Subject: [PATCH 08/95] feat: add deregister and update methods to IServiceRegistry and implement in ServiceRegistryBase --- .../service_registries/IServiceRegistry.py | 14 +++++++++ .../base/ServiceRegistryBase.py | 20 ++++++++++++ .../ServiceRegistry_unit_test.py | 31 +++++++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/pkgs/core/swarmauri_core/service_registries/IServiceRegistry.py b/pkgs/core/swarmauri_core/service_registries/IServiceRegistry.py index e434514aa..8549e52ef 100644 --- a/pkgs/core/swarmauri_core/service_registries/IServiceRegistry.py +++ b/pkgs/core/swarmauri_core/service_registries/IServiceRegistry.py @@ -27,3 +27,17 @@ def get_services_by_roles(self, roles: List[str]) -> List[str]: Get services filtered by their roles. """ pass + + @abstractmethod + def deregister_service(self, name: str) -> None: + """ + Deregister the service with the given name. + """ + pass + + @abstractmethod + def update_service(self, name: str, details: Dict[str, Any]) -> None: + """ + Update the details of the service with the given name. + """ + pass diff --git a/pkgs/swarmauri/swarmauri/service_registries/base/ServiceRegistryBase.py b/pkgs/swarmauri/swarmauri/service_registries/base/ServiceRegistryBase.py index ca6a8824d..6b0b33698 100644 --- a/pkgs/swarmauri/swarmauri/service_registries/base/ServiceRegistryBase.py +++ b/pkgs/swarmauri/swarmauri/service_registries/base/ServiceRegistryBase.py @@ -38,3 +38,23 @@ def get_services_by_roles(self, roles: List[str]) -> List[str]: for name, details in self.services.items() if details.get("role") in roles ] + + def deregister_service(self, name: str) -> None: + """ + Deregister the service with the given name. + """ + if name in self.services: + del self.services[name] + print(f"Service {name} deregistered.") + else: + raise ValueError(f"Service {name} not found.") + + def update_service(self, name: str, details: Dict[str, Any]) -> None: + """ + Update the details of the service with the given name. + """ + if name in self.services: + self.services[name].update(details) + print(f"Service {name} updated with new details: {details}") + else: + raise ValueError(f"Service {name} not found.") diff --git a/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py b/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py index 9c1622a9d..32f122fc6 100644 --- a/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py +++ b/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py @@ -49,3 +49,34 @@ def test_get_services_by_roles(service_registry): assert recipients == ["db"] recipients = service_registry.get_services_by_roles(["authentication", "database"]) assert set(recipients) == {"auth", "db"} + + +@pytest.mark.unit +def test_deregister_service(service_registry): + service_registry.register_service("auth", {"role": "authentication"}) + service_registry.deregister_service("auth") + assert "auth" not in service_registry.services + + +@pytest.mark.unit +def test_deregister_service_nonexistent(service_registry): + with pytest.raises(ValueError) as exc_info: + service_registry.deregister_service("nonexistent") + assert str(exc_info.value) == "Service nonexistent not found." + + +@pytest.mark.unit +def test_update_service(service_registry): + service_registry.register_service("auth", {"role": "authentication"}) + service_registry.update_service("auth", {"role": "auth_service", "version": "1.0"}) + assert service_registry.services["auth"] == { + "role": "auth_service", + "version": "1.0", + } + + +@pytest.mark.unit +def test_update_service_nonexistent(service_registry): + with pytest.raises(ValueError) as exc_info: + service_registry.update_service("nonexistent", {"role": "new_role"}) + assert str(exc_info.value) == "Service nonexistent not found." From 322380e144dd485da0abdcbfa41e7da008ce246e Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Fri, 13 Dec 2024 08:50:48 +0100 Subject: [PATCH 09/95] refactor: rename deregister_service to unregister_service --- .../swarmauri_core/service_registries/IServiceRegistry.py | 4 ++-- .../service_registries/base/ServiceRegistryBase.py | 6 +++--- .../unit/service_registries/ServiceRegistry_unit_test.py | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkgs/core/swarmauri_core/service_registries/IServiceRegistry.py b/pkgs/core/swarmauri_core/service_registries/IServiceRegistry.py index 8549e52ef..911dc7b19 100644 --- a/pkgs/core/swarmauri_core/service_registries/IServiceRegistry.py +++ b/pkgs/core/swarmauri_core/service_registries/IServiceRegistry.py @@ -29,9 +29,9 @@ def get_services_by_roles(self, roles: List[str]) -> List[str]: pass @abstractmethod - def deregister_service(self, name: str) -> None: + def unregister_service(self, name: str) -> None: """ - Deregister the service with the given name. + unregister the service with the given name. """ pass diff --git a/pkgs/swarmauri/swarmauri/service_registries/base/ServiceRegistryBase.py b/pkgs/swarmauri/swarmauri/service_registries/base/ServiceRegistryBase.py index 6b0b33698..a7e567a88 100644 --- a/pkgs/swarmauri/swarmauri/service_registries/base/ServiceRegistryBase.py +++ b/pkgs/swarmauri/swarmauri/service_registries/base/ServiceRegistryBase.py @@ -39,13 +39,13 @@ def get_services_by_roles(self, roles: List[str]) -> List[str]: if details.get("role") in roles ] - def deregister_service(self, name: str) -> None: + def unregister_service(self, name: str) -> None: """ - Deregister the service with the given name. + unregister the service with the given name. """ if name in self.services: del self.services[name] - print(f"Service {name} deregistered.") + print(f"Service {name} unregistered.") else: raise ValueError(f"Service {name} not found.") diff --git a/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py b/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py index 32f122fc6..6498d4c29 100644 --- a/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py +++ b/pkgs/swarmauri/tests/unit/service_registries/ServiceRegistry_unit_test.py @@ -52,16 +52,16 @@ def test_get_services_by_roles(service_registry): @pytest.mark.unit -def test_deregister_service(service_registry): +def test_unregister_service(service_registry): service_registry.register_service("auth", {"role": "authentication"}) - service_registry.deregister_service("auth") + service_registry.unregister_service("auth") assert "auth" not in service_registry.services @pytest.mark.unit -def test_deregister_service_nonexistent(service_registry): +def test_unregister_service_nonexistent(service_registry): with pytest.raises(ValueError) as exc_info: - service_registry.deregister_service("nonexistent") + service_registry.unregister_service("nonexistent") assert str(exc_info.value) == "Service nonexistent not found." From 31cc39650d6ba8566fb48db6a10c736e4bc0f03c Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Fri, 13 Dec 2024 09:31:19 +0100 Subject: [PATCH 10/95] feat: implement lazy loading for ServiceRegistry classes in __init__.py --- .../service_registries/concrete/__init__.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkgs/swarmauri/swarmauri/service_registries/concrete/__init__.py b/pkgs/swarmauri/swarmauri/service_registries/concrete/__init__.py index e69de29bb..09ec2f608 100644 --- a/pkgs/swarmauri/swarmauri/service_registries/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/service_registries/concrete/__init__.py @@ -0,0 +1,16 @@ +from swarmauri.utils._lazy_import import _lazy_import + +# List of service_registry name (file names without the ".py" extension) and corresponding class names +service_registry_files = [ + ( + "swarmauri.service_registries.concrete.ServiceRegistry", + "ServiceRegistry", + ), +] + +# Lazy loading of service_registry classes, storing them in variables +for module_name, class_name in service_registry_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded service_registry classes to __all__ +__all__ = [class_name for _, class_name in service_registry_files] From e7f5a18122b226a70b022cab5e4e1b614bdfbe9e Mon Sep 17 00:00:00 2001 From: 3rd-Son Date: Fri, 13 Dec 2024 12:31:46 +0100 Subject: [PATCH 11/95] finisedh up pubsubtransport --- .../swarmauri/transport/base/TransportBase.py | 4 ++- .../transport/concrete/PubSubTransport.py | 36 +++++++++---------- .../transport/PubSubTransport_unit_test.py | 26 ++++++++++++-- 3 files changed, 43 insertions(+), 23 deletions(-) diff --git a/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py b/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py index 2020f677f..a25cff190 100644 --- a/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py +++ b/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py @@ -1,4 +1,5 @@ from typing import Dict, Any, List, Optional, Literal +from pydantic import ConfigDict, Field from enum import Enum, auto from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes from swarmauri_core.transport.ITransport import ITransport @@ -17,7 +18,8 @@ class TransportationProtocol(Enum): class TransportBase(ITransport, ComponentBase): allowed_protocols: List[TransportationProtocol] = [] - resource: Optional[str] = ResourceTypes.TRANSPORT.value + resource: Optional[str] = Field(default=ResourceTypes.TRANSPORT.value, frozen=True) + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) type: Literal["TransportBase"] = "TransportBase" def send(self, sender: str, recipient: str, message: Any) -> None: diff --git a/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py b/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py index ccc07c221..229114fb7 100644 --- a/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py +++ b/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py @@ -1,24 +1,14 @@ -from typing import Dict, Any, List, Optional, Set +from typing import Dict, Any, List, Optional, Set, Literal from uuid import uuid4 import asyncio -from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes -from swarmauri_core.transport.ITransport import ITransport from swarmauri.transport.base.TransportBase import TransportBase, TransportationProtocol -class PubSubTransport(TransportBase): +class PubSubTransport: allowed_protocols: List[TransportationProtocol] = [TransportationProtocol.PUBSUB] - - def __init__(self): - """ - Initialize the Publish-Subscribe Broker. - Manages topics and subscriptions for agents. - """ - super().__init__() - self._topics: Dict[str, Set[str]] = {} # Topic to subscriber mappings - self._subscribers: Dict[str, asyncio.Queue] = ( - {} - ) # Subscriber ID to message queue + _topics: Dict[str, Set[str]] = {} # Topic to subscriber mappings + _subscribers: Dict[str, asyncio.Queue] = {} + type: Literal["PubSubTransport"] = "PubSubTransport" async def subscribe(self, topic: str) -> str: """ @@ -30,9 +20,9 @@ async def subscribe(self, topic: str) -> str: Returns: str: Unique subscriber ID """ - subscriber_id = str(uuid4()) + subscriber_id = self.id - # Create message queue for this subscriber + # Create message queue for this subscribere self._subscribers[subscriber_id] = asyncio.Queue() # Add subscriber to topic @@ -42,7 +32,7 @@ async def subscribe(self, topic: str) -> str: return subscriber_id - async def unsubscribe(self, topic: str, subscriber_id: str): + async def unsubscribe(self, topic: str): """ Unsubscribe an agent from a topic. @@ -50,6 +40,7 @@ async def unsubscribe(self, topic: str, subscriber_id: str): topic (str): The topic to unsubscribe from subscriber_id (str): Unique identifier of the subscriber """ + subscriber_id = self.id if topic in self._topics and subscriber_id in self._topics[topic]: self._topics[topic].remove(subscriber_id) @@ -72,7 +63,7 @@ async def publish(self, topic: str, message: Any): for subscriber_id in self._topics[topic]: await self._subscribers[subscriber_id].put(message) - async def receive(self, subscriber_id: str) -> Any: + async def receive(self) -> Any: """ Receive messages for a specific subscriber. @@ -82,7 +73,7 @@ async def receive(self, subscriber_id: str) -> Any: Returns: Any: Received message """ - return await self._subscribers[subscriber_id].get() + return await self._subscribers[self.id].get() def send(self, sender: str, recipient: str, message: Any) -> None: """ @@ -120,3 +111,8 @@ def multicast(self, sender: str, recipients: List[str], message: Any) -> None: """ for topic in recipients: asyncio.create_task(self.publish(topic, message)) + + +check = PubSubTransport() +print(check.type) +print("I am okay") diff --git a/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py b/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py index 2d2c68f76..45ef9587c 100644 --- a/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py +++ b/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py @@ -6,12 +6,34 @@ PubSubTransport, ) from swarmauri.utils.timeout_wrapper import timeout +import logging @pytest.fixture -async def pubsub_transport(): +def pubsub_transport(): transport = PubSubTransport() - yield transport + return transport + + +@timeout(5) +@pytest.mark.unit +def test_ubc_resource(pubsub_transport): + assert pubsub_transport.resource == "Transport" + + +@timeout(5) +@pytest.mark.unit +def test_ubc_type(): + assert pubsub_transport.type == "PubSubTransport" + + +@timeout(5) +@pytest.mark.unit +def test_serialization(pubsub_transport): + assert ( + pubsub_transport.id + == PubSubTransport.model_validate_json(pubsub_transport.model_dump_json()).id + ) @timeout(5) From 857bdd0d5fc2bfb61c0a0433e9feeb1b0d49f6f2 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Fri, 13 Dec 2024 06:23:27 -0600 Subject: [PATCH 12/95] Update PubSubTransport.py --- pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py b/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py index 229114fb7..1bf1ea055 100644 --- a/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py +++ b/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py @@ -4,7 +4,7 @@ from swarmauri.transport.base.TransportBase import TransportBase, TransportationProtocol -class PubSubTransport: +class PubSubTransport(TransportBase): allowed_protocols: List[TransportationProtocol] = [TransportationProtocol.PUBSUB] _topics: Dict[str, Set[str]] = {} # Topic to subscriber mappings _subscribers: Dict[str, asyncio.Queue] = {} From 09ae2cb20f05790276c83146fbd387cec9319d9c Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Fri, 13 Dec 2024 06:24:16 -0600 Subject: [PATCH 13/95] Update PubSubTransport_unit_test.py --- .../swarmauri/tests/unit/transport/PubSubTransport_unit_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py b/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py index 45ef9587c..c8918b086 100644 --- a/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py +++ b/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py @@ -23,7 +23,7 @@ def test_ubc_resource(pubsub_transport): @timeout(5) @pytest.mark.unit -def test_ubc_type(): +def test_ubc_type(pubsub_transport): assert pubsub_transport.type == "PubSubTransport" From e77b057ace2d49152078b133b4e63dbdbf0719a5 Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Fri, 13 Dec 2024 13:56:29 +0100 Subject: [PATCH 14/95] feat: add ControlPanel implementation and related interfaces --- pkgs/core/swarmauri_core/ComponentBase.py | 2 + .../control_panels/IControlPanel.py | 43 +++++++++ .../swarmauri_core/control_panels/__init__.py | 0 .../swarmauri/control_panels/__init__.py | 0 .../control_panels/base/ControlPanelBase.py | 62 ++++++++++++ .../swarmauri/control_panels/base/__init__.py | 0 .../control_panels/concrete/ControlPanel.py | 9 ++ .../control_panels/ControlPanel_unit_test.py | 96 +++++++++++++++++++ 8 files changed, 212 insertions(+) create mode 100644 pkgs/core/swarmauri_core/control_panels/IControlPanel.py create mode 100644 pkgs/core/swarmauri_core/control_panels/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/control_panels/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py create mode 100644 pkgs/swarmauri/swarmauri/control_panels/base/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/control_panels/concrete/ControlPanel.py create mode 100644 pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py diff --git a/pkgs/core/swarmauri_core/ComponentBase.py b/pkgs/core/swarmauri_core/ComponentBase.py index a5e558380..b5444fe72 100644 --- a/pkgs/core/swarmauri_core/ComponentBase.py +++ b/pkgs/core/swarmauri_core/ComponentBase.py @@ -58,6 +58,8 @@ class ResourceTypes(Enum): VCM = "VCM" DATA_CONNECTOR = "DataConnector" FACTORY = "Factory" + SERVICE_REGISTRY = "ServiceRegistry" + CONTROL_PANEL = "ControlPanel" def generate_id() -> str: diff --git a/pkgs/core/swarmauri_core/control_panels/IControlPanel.py b/pkgs/core/swarmauri_core/control_panels/IControlPanel.py new file mode 100644 index 000000000..e6fb53191 --- /dev/null +++ b/pkgs/core/swarmauri_core/control_panels/IControlPanel.py @@ -0,0 +1,43 @@ +from abc import ABC, abstractmethod +from typing import Any, List + + +class IControlPlane(ABC): + """ + Abstract base class for ControlPlane. + """ + + @abstractmethod + def create_agent(self, name: str, role: str) -> Any: + """ + Create an agent with the given name and role. + """ + pass + + @abstractmethod + def distribute_tasks(self, task: Any) -> None: + """ + Distribute tasks using the task strategy. + """ + pass + + @abstractmethod + def orchestrate_agents(self, task: Any) -> None: + """ + Orchestrate agents for task distribution. + """ + pass + + @abstractmethod + def remove_agent(self, name: str) -> None: + """ + Remove the agent with the specified name. + """ + pass + + @abstractmethod + def list_active_agents(self) -> List[str]: + """ + List all active agent names. + """ + pass diff --git a/pkgs/core/swarmauri_core/control_panels/__init__.py b/pkgs/core/swarmauri_core/control_panels/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/control_panels/__init__.py b/pkgs/swarmauri/swarmauri/control_panels/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py b/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py new file mode 100644 index 000000000..5202a8406 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py @@ -0,0 +1,62 @@ +from swarmauri_core.control_panels.IControlPanel import IControlPlane +from typing import Any, List, Literal +from pydantic import Field, ConfigDict +from swarmauri_core.ComponentBase import ResourceTypes +from swarmauri.service_registries.base.ServiceRegistryBase import ServiceRegistryBase +from swarmauri.factories.base.FactoryBase import FactoryBase +from swarmauri.task_strategies.base.TaskStrategyBase import TaskStrategyBase +from swarmauri.transports.base.TransportBase import TransportBase +from swarmauri_core.typing import SubclassUnion + + +class ControlPanelBase(IControlPlane): + """ + Implementation of the ControlPlane abstract class. + """ + + resource: ResourceTypes = Field(default=ResourceTypes.CONTROL_PANEL.value) + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + type: Literal["ControlPanelBase"] = "ControlPanelBase" + + agent_factory: SubclassUnion[FactoryBase] + service_registry: SubclassUnion[ServiceRegistryBase] + task_strategy: SubclassUnion[TaskStrategyBase] + transport: SubclassUnion[TransportBase] + + def create_agent(self, name: str, role: str) -> Any: + """ + Create an agent with the given name and role. + """ + agent = self.agent_factory.create_agent(name, role) + self.service_registry.register_service(name, {"role": role, "status": "active"}) + return agent + + def distribute_tasks(self, task: Any) -> None: + """ + Distribute tasks using the task strategy. + """ + self.task_strategy.assign_task(task, self.agent_factory, self.service_registry) + + def orchestrate_agents(self, task: Any) -> None: + """ + Orchestrate agents for task distribution. + """ + self.manage_agents() + self.distribute_tasks(task) + + def remove_agent(self, name: str) -> None: + """ + Remove the agent with the specified name. + """ + agent = self.agent_factory.get_agent_by_name(name) + if not agent: + raise ValueError(f"Agent {name} not found.") + self.service_registry.unregister_service(name) + self.agent_factory.delete_agent(name) + + def list_active_agents(self) -> List[str]: + """ + List all active agent names. + """ + agents = self.agent_factory.get_agents() + return [agent.name for agent in agents if agent] diff --git a/pkgs/swarmauri/swarmauri/control_panels/base/__init__.py b/pkgs/swarmauri/swarmauri/control_panels/base/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/control_panels/concrete/ControlPanel.py b/pkgs/swarmauri/swarmauri/control_panels/concrete/ControlPanel.py new file mode 100644 index 000000000..c8533ebc4 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/control_panels/concrete/ControlPanel.py @@ -0,0 +1,9 @@ +from typing import Literal +from swarmauri.control_panels.base.ControlPanelBase import ControlPanelBase + +class ControlPanel(ControlPanelBase): + """ + Concrete implementation of the ControlPanelBase. + """ + type: Literal["ControlPanel"] = "ControlPanel" + diff --git a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py new file mode 100644 index 000000000..83e180e6c --- /dev/null +++ b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py @@ -0,0 +1,96 @@ +import pytest +from unittest.mock import MagicMock, patch +from swarmauri.control_panels.concrete.ControlPanel import ControlPanel + + +@pytest.fixture +def control_panel(): + agent_factory = MagicMock() + service_registry = MagicMock() + task_strategy = MagicMock() + transport = MagicMock() + return ControlPanel(agent_factory, service_registry, task_strategy, transport) + + +def test_create_agent(control_panel): + # Arrange + control_panel.agent_factory.create_agent.return_value = "agent1" + name = "agent1" + role = "role1" + + # Act + result = control_panel.create_agent(name, role) + + # Assert + control_panel.agent_factory.create_agent.assert_called_with(name, role) + control_panel.service_registry.register_service.assert_called_with( + name, {"role": role, "status": "active"} + ) + assert result == "agent1" + + +def test_distribute_tasks(control_panel): + # Arrange + task = "task1" + + # Act + control_panel.distribute_tasks(task) + + # Assert + control_panel.task_strategy.assign_task.assert_called_with( + task, control_panel.agent_factory, control_panel.service_registry + ) + + +def test_orchestrate_agents(control_panel): + # Arrange + task = "task1" + + # Act + control_panel.orchestrate_agents(task) + + # Assert + with patch.object(control_panel, "manage_agents") as mock_manage_agents: + mock_manage_agents.assert_called_once() + control_panel.distribute_tasks.assert_called_with(task) + + +def test_remove_agent_success(control_panel): + # Arrange + name = "agent1" + agent = MagicMock() + control_panel.agent_factory.get_agent_by_name.return_value = agent + + # Act + control_panel.remove_agent(name) + + # Assert + control_panel.service_registry.unregister_service.assert_called_with(name) + control_panel.agent_factory.delete_agent.assert_called_with(name) + + +def test_remove_agent_not_found(control_panel): + # Arrange + name = "agent1" + control_panel.agent_factory.get_agent_by_name.return_value = None + + # Act & Assert + with pytest.raises(ValueError) as excinfo: + control_panel.remove_agent(name) + assert str(excinfo.value) == f"Agent {name} not found." + + +def test_list_active_agents(control_panel): + # Arrange + agent1 = MagicMock() + agent1.name = "agent1" + agent2 = MagicMock() + agent2.name = "agent2" + control_panel.agent_factory.get_agents.return_value = [agent1, agent2] + + # Act + result = control_panel.list_active_agents() + + # Assert + control_panel.agent_factory.get_agents.assert_called_once() + assert result == ["agent1", "agent2"] From a57dc3dd528609592b8c81f2ef18dbf7dc801d68 Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Mon, 16 Dec 2024 10:43:00 +0100 Subject: [PATCH 15/95] feat: implement task management strategies with base and round-robin strategy --- pkgs/core/swarmauri_core/ComponentBase.py | 1 + .../task_mgt_strategies/ITaskMgtStrategy.py | 43 +++++++++ .../task_mgt_strategies/__init__.py | 0 .../control_panels/base/ControlPanelBase.py | 7 +- .../control_panels/concrete/ControlPanel.py | 3 +- .../swarmauri/task_mgt_strategies/__init__.py | 0 .../base/TaskMgtStrategyBase.py | 63 +++++++++++++ .../task_mgt_strategies/base/__init__.py | 0 .../concrete/RoundRobinStrategy.py | 70 ++++++++++++++ .../task_mgt_strategies/concrete/__init__.py | 0 .../control_panels/ControlPanel_unit_test.py | 13 --- .../RoundRobinStrategy_unit_test.py | 93 +++++++++++++++++++ 12 files changed, 275 insertions(+), 18 deletions(-) create mode 100644 pkgs/core/swarmauri_core/task_mgt_strategies/ITaskMgtStrategy.py create mode 100644 pkgs/core/swarmauri_core/task_mgt_strategies/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/task_mgt_strategies/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/task_mgt_strategies/base/TaskMgtStrategyBase.py create mode 100644 pkgs/swarmauri/swarmauri/task_mgt_strategies/base/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/RoundRobinStrategy.py create mode 100644 pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/__init__.py create mode 100644 pkgs/swarmauri/tests/unit/task_mgt_strategies/RoundRobinStrategy_unit_test.py diff --git a/pkgs/core/swarmauri_core/ComponentBase.py b/pkgs/core/swarmauri_core/ComponentBase.py index f9be2bdcb..a72e62fac 100644 --- a/pkgs/core/swarmauri_core/ComponentBase.py +++ b/pkgs/core/swarmauri_core/ComponentBase.py @@ -60,6 +60,7 @@ class ResourceTypes(Enum): FACTORY = "Factory" SERVICE_REGISTRY = "ServiceRegistry" CONTROL_PANEL = "ControlPanel" + TASK_MGT_STRATEGY = "TaskMgtStrategy" def generate_id() -> str: diff --git a/pkgs/core/swarmauri_core/task_mgt_strategies/ITaskMgtStrategy.py b/pkgs/core/swarmauri_core/task_mgt_strategies/ITaskMgtStrategy.py new file mode 100644 index 000000000..13be06917 --- /dev/null +++ b/pkgs/core/swarmauri_core/task_mgt_strategies/ITaskMgtStrategy.py @@ -0,0 +1,43 @@ +from abc import ABC, abstractmethod +from typing import Any, Callable, Dict + + +class ITaskMgtStrategy(ABC): + """Abstract base class for TaskStrategy.""" + + @abstractmethod + def assign_task( + self, task: Dict[str, Any], agent_factory: Callable, service_registry: Callable + ) -> str: + """ + Abstract method to assign a task to a service. + """ + pass + + @abstractmethod + def add_task(self, task: Dict[str, Any]) -> None: + """ + Abstract method to add a task to the task queue. + """ + pass + + @abstractmethod + def remove_task(self, task_id: str) -> None: + """ + Abstract method to remove a task from the task queue. + """ + pass + + @abstractmethod + def get_task(self, task_id: str) -> Dict[str, Any]: + """ + Abstract method to get a task from the task queue. + """ + pass + + @abstractmethod + def process_tasks(self, task: Dict[str, Any]) -> None: + """ + Abstract method to process a task. + """ + pass diff --git a/pkgs/core/swarmauri_core/task_mgt_strategies/__init__.py b/pkgs/core/swarmauri_core/task_mgt_strategies/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py b/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py index 5202a8406..cee5b6c90 100644 --- a/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py +++ b/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py @@ -4,7 +4,7 @@ from swarmauri_core.ComponentBase import ResourceTypes from swarmauri.service_registries.base.ServiceRegistryBase import ServiceRegistryBase from swarmauri.factories.base.FactoryBase import FactoryBase -from swarmauri.task_strategies.base.TaskStrategyBase import TaskStrategyBase +from swarmauri.task_mgt_strategies.base.TaskMgtStrategyBase import TaskMgtStrategyBase from swarmauri.transports.base.TransportBase import TransportBase from swarmauri_core.typing import SubclassUnion @@ -20,7 +20,7 @@ class ControlPanelBase(IControlPlane): agent_factory: SubclassUnion[FactoryBase] service_registry: SubclassUnion[ServiceRegistryBase] - task_strategy: SubclassUnion[TaskStrategyBase] + task_mgt_strategy: SubclassUnion[TaskMgtStrategyBase] transport: SubclassUnion[TransportBase] def create_agent(self, name: str, role: str) -> Any: @@ -35,13 +35,12 @@ def distribute_tasks(self, task: Any) -> None: """ Distribute tasks using the task strategy. """ - self.task_strategy.assign_task(task, self.agent_factory, self.service_registry) + self.task_mgt_strategy.assign_task(task, self.service_registry) def orchestrate_agents(self, task: Any) -> None: """ Orchestrate agents for task distribution. """ - self.manage_agents() self.distribute_tasks(task) def remove_agent(self, name: str) -> None: diff --git a/pkgs/swarmauri/swarmauri/control_panels/concrete/ControlPanel.py b/pkgs/swarmauri/swarmauri/control_panels/concrete/ControlPanel.py index c8533ebc4..14a7b2205 100644 --- a/pkgs/swarmauri/swarmauri/control_panels/concrete/ControlPanel.py +++ b/pkgs/swarmauri/swarmauri/control_panels/concrete/ControlPanel.py @@ -1,9 +1,10 @@ from typing import Literal from swarmauri.control_panels.base.ControlPanelBase import ControlPanelBase + class ControlPanel(ControlPanelBase): """ Concrete implementation of the ControlPanelBase. """ + type: Literal["ControlPanel"] = "ControlPanel" - diff --git a/pkgs/swarmauri/swarmauri/task_mgt_strategies/__init__.py b/pkgs/swarmauri/swarmauri/task_mgt_strategies/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/task_mgt_strategies/base/TaskMgtStrategyBase.py b/pkgs/swarmauri/swarmauri/task_mgt_strategies/base/TaskMgtStrategyBase.py new file mode 100644 index 000000000..698751f71 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/task_mgt_strategies/base/TaskMgtStrategyBase.py @@ -0,0 +1,63 @@ +from abc import abstractmethod + +from pydantic import ConfigDict, Field +from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes +from swarmauri_core.task_mgt_strategies.ITaskMgtStrategy import ITaskMgtStrategy +from typing import Any, Callable, Dict, Literal, Optional + + +class TaskMgtStrategyBase(ITaskMgtStrategy, ComponentBase): + """Base class for TaskStrategy.""" + + type: Literal["TaskMgtStrategyBase"] = "TaskMgtStrategyBase" + resource: Optional[str] = Field( + default=ResourceTypes.TASK_MGT_STRATEGY.value, frozen=True + ) + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + + @abstractmethod + def assign_task( + self, task: Dict[str, Any], agent_factory: Callable, service_registry: Callable + ) -> str: + """ + Abstract method to assign a task to a service. + """ + raise NotImplementedError( + "assign_task method must be implemented in derived classes." + ) + + @abstractmethod + def add_task(self, task: Dict[str, Any]) -> None: + """ + Abstract method to add a task to the task queue. + """ + raise NotImplementedError( + "add_task method must be implemented in derived classes." + ) + + @abstractmethod + def remove_task(self, task_id: str) -> None: + """ + Abstract method to remove a task from the task queue. + """ + raise NotImplementedError( + "remove_task method must be implemented in derived classes." + ) + + @abstractmethod + def get_task(self, task_id: str) -> Dict[str, Any]: + """ + Abstract method to get a task from the task queue. + """ + raise NotImplementedError( + "get_task method must be implemented in derived classes." + ) + + @abstractmethod + def process_tasks(self, task: Dict[str, Any]) -> None: + """ + Abstract method to process tasks. + """ + raise NotImplementedError( + "process_task method must be implemented in derived classes." + ) diff --git a/pkgs/swarmauri/swarmauri/task_mgt_strategies/base/__init__.py b/pkgs/swarmauri/swarmauri/task_mgt_strategies/base/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/RoundRobinStrategy.py b/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/RoundRobinStrategy.py new file mode 100644 index 000000000..53b1bddd6 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/RoundRobinStrategy.py @@ -0,0 +1,70 @@ +from typing import Callable, Dict, Any, List +from swarmauri.task_mgt_strategies.base.TaskMgtStrategyBase import TaskMgtStrategyBase +from queue import Queue +import logging + + +class RoundRobinStrategy(TaskMgtStrategyBase): + """Round-robin task assignment strategy.""" + + task_queue: Queue = Queue() # Synchronous task queue for incoming tasks + task_assignments: Dict[str, str] = {} # Tracks task assignments + current_index: int = 0 # Tracks the next service to assign tasks to + + def assign_task(self, task: Dict[str, Any], service_registry: Callable[[], List[str]]) -> None: + """ + Assign a task to a service using the round-robin strategy. + :param task: Task metadata and payload. + :param service_registry: Callable that returns available services. + """ + available_services = service_registry() + if not available_services: + raise ValueError("No services available for task assignment.") + + # Select the service based on the round-robin index + service = available_services[self.current_index % len(available_services)] + self.task_assignments[task["task_id"]] = service + self.current_index += 1 + logging.info(f"Task '{task['task_id']}' assigned to service '{service}'.") + + def add_task(self, task: Dict[str, Any]) -> None: + """ + Add a task to the task queue. + :param task: Task metadata and payload. + """ + self.task_queue.put(task) + + def remove_task(self, task_id: str) -> None: + """ + Remove a task from the task registry. + :param task_id: Unique identifier of the task to remove. + """ + if task_id in self.task_assignments: + del self.task_assignments[task_id] + logging.info(f"Task '{task_id}' removed from assignments.") + else: + raise ValueError(f"Task '{task_id}' not found in assignments.") + + def get_task(self, task_id: str) -> Dict[str, Any]: + """ + Get a task's assigned service. + :param task_id: Unique identifier of the task. + :return: Task assignment details. + """ + if task_id in self.task_assignments: + service = self.task_assignments[task_id] + return {"task_id": task_id, "assigned_service": service} + else: + raise ValueError(f"Task '{task_id}' not found in assignments.") + + def process_tasks(self, service_registry: Callable[[], List[str]]) -> None: + """ + Process tasks from the task queue and assign them to services. + :param service_registry: Callable that returns available services. + """ + while not self.task_queue.empty(): + task = self.task_queue.get() + try: + self.assign_task(task, service_registry) + except ValueError as e: + raise ValueError(f"Error assigning task: {e}") diff --git a/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/__init__.py b/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py index 83e180e6c..327f97e34 100644 --- a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py +++ b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py @@ -13,15 +13,12 @@ def control_panel(): def test_create_agent(control_panel): - # Arrange control_panel.agent_factory.create_agent.return_value = "agent1" name = "agent1" role = "role1" - # Act result = control_panel.create_agent(name, role) - # Assert control_panel.agent_factory.create_agent.assert_called_with(name, role) control_panel.service_registry.register_service.assert_called_with( name, {"role": role, "status": "active"} @@ -30,26 +27,20 @@ def test_create_agent(control_panel): def test_distribute_tasks(control_panel): - # Arrange task = "task1" - # Act control_panel.distribute_tasks(task) - # Assert control_panel.task_strategy.assign_task.assert_called_with( task, control_panel.agent_factory, control_panel.service_registry ) def test_orchestrate_agents(control_panel): - # Arrange task = "task1" - # Act control_panel.orchestrate_agents(task) - # Assert with patch.object(control_panel, "manage_agents") as mock_manage_agents: mock_manage_agents.assert_called_once() control_panel.distribute_tasks.assert_called_with(task) @@ -61,10 +52,8 @@ def test_remove_agent_success(control_panel): agent = MagicMock() control_panel.agent_factory.get_agent_by_name.return_value = agent - # Act control_panel.remove_agent(name) - # Assert control_panel.service_registry.unregister_service.assert_called_with(name) control_panel.agent_factory.delete_agent.assert_called_with(name) @@ -88,9 +77,7 @@ def test_list_active_agents(control_panel): agent2.name = "agent2" control_panel.agent_factory.get_agents.return_value = [agent1, agent2] - # Act result = control_panel.list_active_agents() - # Assert control_panel.agent_factory.get_agents.assert_called_once() assert result == ["agent1", "agent2"] diff --git a/pkgs/swarmauri/tests/unit/task_mgt_strategies/RoundRobinStrategy_unit_test.py b/pkgs/swarmauri/tests/unit/task_mgt_strategies/RoundRobinStrategy_unit_test.py new file mode 100644 index 000000000..d28d9ab3c --- /dev/null +++ b/pkgs/swarmauri/tests/unit/task_mgt_strategies/RoundRobinStrategy_unit_test.py @@ -0,0 +1,93 @@ +import pytest +from swarmauri.task_mgt_strategies.concrete.RoundRobinStrategy import RoundRobinStrategy +from unittest.mock import MagicMock + + +@pytest.fixture +def round_robin_strategy(): + """Fixture to create a RoundRobinStrategy instance.""" + return RoundRobinStrategy() + + +def test_assign_task(round_robin_strategy): + # Setup + task = {"task_id": "task1", "payload": "data"} + service_registry = MagicMock(return_value=["service1", "service2"]) + + round_robin_strategy.assign_task(task, service_registry) + + assert round_robin_strategy.task_assignments["task1"] == "service1" + assert round_robin_strategy.current_index == 1 + + +def test_assign_task_no_services(round_robin_strategy): + # Setup + task = {"task_id": "task1", "payload": "data"} + service_registry = MagicMock(return_value=[]) + + # Execute & Verify + with pytest.raises(ValueError) as exc_info: + round_robin_strategy.assign_task(task, service_registry) + assert str(exc_info.value) == "No services available for task assignment." + + +def test_add_task(round_robin_strategy): + task = {"task_id": "task1", "payload": "data"} + + round_robin_strategy.add_task(task) + + assert not round_robin_strategy.task_queue.empty() + queued_task = round_robin_strategy.task_queue.get() + assert queued_task == task + + +def test_remove_task(round_robin_strategy): + task_id = "task1" + round_robin_strategy.task_assignments[task_id] = "service1" + + round_robin_strategy.remove_task(task_id) + + assert task_id not in round_robin_strategy.task_assignments + + +def test_remove_task_not_found(round_robin_strategy): + task_id = "task1" + + with pytest.raises(ValueError) as exc_info: + round_robin_strategy.remove_task(task_id) + assert str(exc_info.value) == "Task 'task1' not found in assignments." + + +def test_get_task(round_robin_strategy): + task_id = "task1" + round_robin_strategy.task_assignments[task_id] = "service1" + + result = round_robin_strategy.get_task(task_id) + + assert result == {"task_id": task_id, "assigned_service": "service1"} + + +def test_get_task_not_found(round_robin_strategy): + task_id = "task1" + + with pytest.raises(ValueError) as exc_info: + round_robin_strategy.get_task(task_id) + assert str(exc_info.value) == "Task 'task1' not found in assignments." + + +def test_process_tasks(round_robin_strategy): + service_registry = MagicMock(return_value=["service1", "service2"]) + tasks = [ + {"task_id": "task1", "payload": "data1"}, + {"task_id": "task2", "payload": "data2"}, + {"task_id": "task3", "payload": "data3"}, + ] + for task in tasks: + round_robin_strategy.add_task(task) + + round_robin_strategy.process_tasks(service_registry) + + assert round_robin_strategy.task_assignments["task1"] == "service1" + assert round_robin_strategy.task_assignments["task2"] == "service2" + assert round_robin_strategy.task_assignments["task3"] == "service1" + assert round_robin_strategy.current_index == 3 From de249fc5a8f3fd8310b9e4c26d559382168d6034 Mon Sep 17 00:00:00 2001 From: 3rd-Son Date: Mon, 16 Dec 2024 12:27:25 +0100 Subject: [PATCH 16/95] implemented pipelines --- pkgs/core/swarmauri_core/ComponentBase.py | 1 + .../swarmauri_core/pipelines/IPipeline.py | 57 ++++++++++ .../core/swarmauri_core/pipelines/__init__.py | 0 .../swarmauri/pipelines/base/PipelineBase.py | 105 ++++++++++++++++++ .../swarmauri/pipelines/base/__init__.py | 0 .../swarmauri/pipelines/concrete/Pipeline.py | 50 +++++++++ .../swarmauri/pipelines/concrete/__init__.py | 13 +++ .../unit/pipelines/Pipeline_unit_test.py | 84 ++++++++++++++ 8 files changed, 310 insertions(+) create mode 100644 pkgs/core/swarmauri_core/pipelines/IPipeline.py create mode 100644 pkgs/core/swarmauri_core/pipelines/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/pipelines/base/PipelineBase.py create mode 100644 pkgs/swarmauri/swarmauri/pipelines/base/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/pipelines/concrete/Pipeline.py create mode 100644 pkgs/swarmauri/swarmauri/pipelines/concrete/__init__.py create mode 100644 pkgs/swarmauri/tests/unit/pipelines/Pipeline_unit_test.py diff --git a/pkgs/core/swarmauri_core/ComponentBase.py b/pkgs/core/swarmauri_core/ComponentBase.py index ed142c918..2513f0d88 100644 --- a/pkgs/core/swarmauri_core/ComponentBase.py +++ b/pkgs/core/swarmauri_core/ComponentBase.py @@ -58,6 +58,7 @@ class ResourceTypes(Enum): VCM = "VCM" DATA_CONNECTOR = "DataConnector" FACTORY = "Factory" + PIPELINE = "Pipeline" def generate_id() -> str: diff --git a/pkgs/core/swarmauri_core/pipelines/IPipeline.py b/pkgs/core/swarmauri_core/pipelines/IPipeline.py new file mode 100644 index 000000000..9e941fdd9 --- /dev/null +++ b/pkgs/core/swarmauri_core/pipelines/IPipeline.py @@ -0,0 +1,57 @@ +from abc import ABC, abstractmethod +from typing import Any, Callable, List +from enum import Enum + + +class PipelineStatus(Enum): + """ + Enum representing the status of a pipeline execution. + """ + + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + STOPPED = "stopped" + + +class IPipeline(ABC): + """ + Interface defining core methods for pipeline execution and management. + """ + + @abstractmethod + def add_task(self, task: Callable, *args: Any, **kwargs: Any) -> None: + """ + Add a task to the pipeline. + + :param task: Callable task to be executed + :param args: Positional arguments for the task + :param kwargs: Keyword arguments for the task + """ + pass + + @abstractmethod + def execute(self, *args: Any, **kwargs: Any) -> List[Any]: + """ + Execute the entire pipeline. + + :return: List of results from pipeline execution + """ + pass + + @abstractmethod + def get_status(self) -> PipelineStatus: + """ + Get the current status of the pipeline. + + :return: Current pipeline status + """ + pass + + @abstractmethod + def reset(self) -> None: + """ + Reset the pipeline to its initial state. + """ + pass diff --git a/pkgs/core/swarmauri_core/pipelines/__init__.py b/pkgs/core/swarmauri_core/pipelines/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/pipelines/base/PipelineBase.py b/pkgs/swarmauri/swarmauri/pipelines/base/PipelineBase.py new file mode 100644 index 000000000..da91427b9 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/pipelines/base/PipelineBase.py @@ -0,0 +1,105 @@ +from typing import Any, Callable, List, Optional, Dict +from pydantic import BaseModel, ConfigDict, Field +from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes +from swarmauri_core.pipelines.IPipeline import IPipeline, PipelineStatus +import uuid + + +class PipelineBase(ComponentBase, IPipeline): + """ + Base class providing default behavior for task orchestration, + error handling, and result aggregation. + """ + + resource: Optional[str] = Field(default=ResourceTypes.PIPELINE.value, frozen=True) + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + type: str = "PipelineBase" + + # Pydantic model fields + id: str = Field(default_factory=lambda: str(uuid.uuid4())) + tasks: List[Dict[str, Any]] = Field(default_factory=list) + parallel: bool = Field(default=False) + + def __init__( + self, tasks: Optional[List[Dict[str, Any]]] = None, parallel: bool = False + ): + """ + Initialize the pipeline. + + :param tasks: Optional list of tasks to initialize pipeline with + :param parallel: Flag to indicate parallel or sequential execution + """ + super().__init__() + self.tasks = tasks or [] + self._results: List[Any] = [] + self._status: PipelineStatus = PipelineStatus.PENDING + self.parallel = parallel + + def add_task(self, task: Callable, *args: Any, **kwargs: Any) -> None: + """ + Add a task to the pipeline. + + :param task: Callable task to be executed + :param args: Positional arguments for the task + :param kwargs: Keyword arguments for the task + """ + task_entry = {"callable": task, "args": args, "kwargs": kwargs} + self.tasks.append(task_entry) + + def execute(self, *args: Any, **kwargs: Any) -> List[Any]: + """ + Execute pipeline tasks. + + :return: List of results from pipeline execution + """ + try: + self._status = PipelineStatus.RUNNING + self._results = [] + + if self.parallel: + # Implement parallel execution logic + from concurrent.futures import ThreadPoolExecutor + + with ThreadPoolExecutor() as executor: + futures = [ + executor.submit( + task["callable"], *task["args"], **task["kwargs"] + ) + for task in self.tasks + ] + self._results = [future.result() for future in futures] + else: + # Sequential execution + for task in self.tasks: + result = task["callable"](*task["args"], **task["kwargs"]) + self._results.append(result) + + self._status = PipelineStatus.COMPLETED + return self._results + + except Exception as e: + self._status = PipelineStatus.FAILED + raise RuntimeError(f"Pipeline execution failed: {e}") + + def get_status(self) -> PipelineStatus: + """ + Get the current status of the pipeline. + + :return: Current pipeline status + """ + return self._status + + def reset(self) -> None: + """ + Reset the pipeline to its initial state. + """ + self._results = [] + self._status = PipelineStatus.PENDING + + def get_results(self) -> List[Any]: + """ + Get the results of the pipeline execution. + + :return: List of results + """ + return self._results diff --git a/pkgs/swarmauri/swarmauri/pipelines/base/__init__.py b/pkgs/swarmauri/swarmauri/pipelines/base/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/pipelines/concrete/Pipeline.py b/pkgs/swarmauri/swarmauri/pipelines/concrete/Pipeline.py new file mode 100644 index 000000000..82c4ec593 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/pipelines/concrete/Pipeline.py @@ -0,0 +1,50 @@ +from typing import Any, Callable, List, Optional, Dict +from swarmauri.pipelines.base.PipelineBase import PipelineBase + + +class Pipeline(PipelineBase): + """ + Concrete implementation of a pipeline with additional + customization options. + """ + + type: str = "Pipeline" + + def __init__( + self, + tasks: Optional[List[Dict[str, Any]]] = None, + parallel: bool = False, + error_handler: Optional[Callable[[Exception], Any]] = None, + ): + """ + Initialize a customizable pipeline. + + :param tasks: Optional list of tasks to initialize pipeline with + :param parallel: Flag to indicate parallel or sequential execution + :param error_handler: Optional custom error handling function + """ + super().__init__(tasks, parallel) + self._error_handler = error_handler + + def execute(self, *args: Any, **kwargs: Any) -> List[Any]: + """ + Execute pipeline with optional custom error handling. + + :return: List of results from pipeline execution + """ + try: + return super().execute(*args, **kwargs) + except Exception as e: + if self._error_handler: + return [self._error_handler(e)] + raise + + def with_error_handler(self, handler: Callable[[Exception], Any]) -> "Pipeline": + """ + Add a custom error handler to the pipeline. + + :param handler: Error handling function + :return: Current pipeline instance + """ + self._error_handler = handler + return self diff --git a/pkgs/swarmauri/swarmauri/pipelines/concrete/__init__.py b/pkgs/swarmauri/swarmauri/pipelines/concrete/__init__.py new file mode 100644 index 000000000..db1142f92 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/pipelines/concrete/__init__.py @@ -0,0 +1,13 @@ +from swarmauri.utils._lazy_import import _lazy_import + +# List of pipeline names (file names without the ".py" extension) and corresponding class names +pipeline_files = [ + ("swarmauri.pipelines.concrete.Pipeline", "Pipeline"), +] + +# Lazy loading of pipeline classes, storing them in variables +for module_name, class_name in pipeline_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded pipeline classes to __all__ +__all__ = [class_name for _, class_name in pipeline_files] diff --git a/pkgs/swarmauri/tests/unit/pipelines/Pipeline_unit_test.py b/pkgs/swarmauri/tests/unit/pipelines/Pipeline_unit_test.py new file mode 100644 index 000000000..359d9d759 --- /dev/null +++ b/pkgs/swarmauri/tests/unit/pipelines/Pipeline_unit_test.py @@ -0,0 +1,84 @@ +import pytest +from swarmauri.pipelines.concrete.Pipeline import Pipeline +from swarmauri_core.pipelines.IPipeline import PipelineStatus + + +@pytest.fixture(scope="module") +def simple_tasks(): + def task1(): + return "Task 1 completed" + + def task2(x): + return f"Task 2 with {x}" + + def task3(x, y): + return x + y + + return [task1, task2, task3] + + +@pytest.fixture(scope="module") +def pipeline(simple_tasks): + pipeline = Pipeline() + pipeline.add_task(simple_tasks[0]) + pipeline.add_task(simple_tasks[1], "parameter") + pipeline.add_task(simple_tasks[2], 10, 20) + return pipeline + + +@pytest.mark.unit +def test_ubc_resource(pipeline): + assert pipeline.resource == "Pipeline" + + +@pytest.mark.unit +def test_ubc_type(pipeline): + assert pipeline.type == "Pipeline" + + +@pytest.mark.unit +def test_serialization(pipeline): + assert pipeline.id == Pipeline.model_validate_json(pipeline.model_dump_json()).id + + +@pytest.mark.unit +def test_pipeline_initial_status(pipeline): + assert pipeline.get_status() == PipelineStatus.PENDING + + +@pytest.mark.unit +def test_pipeline_execution(pipeline): + results = pipeline.execute() + + assert len(results) == 3 + assert results[0] == "Task 1 completed" + assert results[1] == "Task 2 with parameter" + assert results[2] == 30 + assert pipeline.get_status() == PipelineStatus.COMPLETED + + +@pytest.mark.unit +def test_pipeline_reset(pipeline): + pipeline.reset() + assert pipeline.get_status() == PipelineStatus.PENDING + assert len(pipeline.get_results()) == 0 + + +@pytest.mark.unit +def test_pipeline_add_task(simple_tasks): + pipeline = Pipeline() + initial_task_count = len(pipeline.tasks) + + pipeline.add_task(simple_tasks[0]) + assert len(pipeline.tasks) == initial_task_count + 1 + + +@pytest.mark.unit +def test_pipeline_get_results(simple_tasks): + pipeline = Pipeline() + pipeline.add_task(simple_tasks[0]) + pipeline.execute() + + results = pipeline.get_results() + assert len(results) == 1 + assert results[0] == "Task 1 completed" From 3f0c3dc439b93591a494a20c75b765a191abcf46 Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Mon, 16 Dec 2024 13:21:55 +0100 Subject: [PATCH 17/95] feat: enhance RoundRobinStrategy and ControlPanelBase with task processing and agent management improvements --- .../control_panels/IControlPanel.py | 30 ++- .../control_panels/base/ControlPanelBase.py | 70 +++++-- .../concrete/RoundRobinStrategy.py | 5 +- .../control_panels/ControlPanel_unit_test.py | 179 +++++++++++++----- .../RoundRobinStrategy_unit_test.py | 45 ++++- 5 files changed, 253 insertions(+), 76 deletions(-) diff --git a/pkgs/core/swarmauri_core/control_panels/IControlPanel.py b/pkgs/core/swarmauri_core/control_panels/IControlPanel.py index e6fb53191..d1810b023 100644 --- a/pkgs/core/swarmauri_core/control_panels/IControlPanel.py +++ b/pkgs/core/swarmauri_core/control_panels/IControlPanel.py @@ -15,29 +15,43 @@ def create_agent(self, name: str, role: str) -> Any: pass @abstractmethod - def distribute_tasks(self, task: Any) -> None: + def remove_agent(self, name: str) -> None: """ - Distribute tasks using the task strategy. + Remove the agent with the specified name. """ pass @abstractmethod - def orchestrate_agents(self, task: Any) -> None: + def list_active_agents(self) -> List[str]: """ - Orchestrate agents for task distribution. + List all active agent names. """ pass @abstractmethod - def remove_agent(self, name: str) -> None: + def submit_tasks(self, tasks: List[Any]) -> None: """ - Remove the agent with the specified name. + Submit one or more tasks to the task management strategy for processing. """ pass @abstractmethod - def list_active_agents(self) -> List[str]: + def process_tasks(self) -> None: """ - List all active agent names. + Process and assign tasks from the queue, then transport them to their assigned services. + """ + pass + + @abstractmethod + def distribute_tasks(self, task: Any) -> None: + """ + Distribute tasks using the task strategy. + """ + pass + + @abstractmethod + def orchestrate_agents(self, task: Any) -> None: + """ + Orchestrate agents for task distribution. """ pass diff --git a/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py b/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py index cee5b6c90..41653005c 100644 --- a/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py +++ b/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py @@ -7,11 +7,14 @@ from swarmauri.task_mgt_strategies.base.TaskMgtStrategyBase import TaskMgtStrategyBase from swarmauri.transports.base.TransportBase import TransportBase from swarmauri_core.typing import SubclassUnion +import logging class ControlPanelBase(IControlPlane): """ Implementation of the ControlPlane abstract class. + This class orchestrates agents, manages tasks, and ensures task distribution + and transport between agents and services. """ resource: ResourceTypes = Field(default=ResourceTypes.CONTROL_PANEL.value) @@ -23,39 +26,74 @@ class ControlPanelBase(IControlPlane): task_mgt_strategy: SubclassUnion[TaskMgtStrategyBase] transport: SubclassUnion[TransportBase] + # Agent management methods def create_agent(self, name: str, role: str) -> Any: """ - Create an agent with the given name and role. + Create an agent with the given name and role, and register it in the service registry. """ agent = self.agent_factory.create_agent(name, role) self.service_registry.register_service(name, {"role": role, "status": "active"}) + logging.info(f"Agent '{name}' with role '{role}' created and registered.") return agent - def distribute_tasks(self, task: Any) -> None: - """ - Distribute tasks using the task strategy. - """ - self.task_mgt_strategy.assign_task(task, self.service_registry) - - def orchestrate_agents(self, task: Any) -> None: - """ - Orchestrate agents for task distribution. - """ - self.distribute_tasks(task) - def remove_agent(self, name: str) -> None: """ - Remove the agent with the specified name. + Remove the agent with the specified name and unregister it from the service registry. """ agent = self.agent_factory.get_agent_by_name(name) if not agent: - raise ValueError(f"Agent {name} not found.") + raise ValueError(f"Agent '{name}' not found.") self.service_registry.unregister_service(name) self.agent_factory.delete_agent(name) + logging.info(f"Agent '{name}' removed and unregistered.") def list_active_agents(self) -> List[str]: """ List all active agent names. """ agents = self.agent_factory.get_agents() - return [agent.name for agent in agents if agent] + active_agents = [agent.name for agent in agents if agent] + logging.info(f"Active agents listed: {active_agents}") + return active_agents + + # Task management methods + def submit_tasks(self, tasks: List[Any]) -> None: + """ + Submit one or more tasks to the task management strategy for processing. + """ + for task in tasks: + self.task_mgt_strategy.add_task(task) + logging.info( + f"Task '{task.get('task_id', 'unknown')}' submitted to the strategy." + ) + + def process_tasks(self) -> None: + """ + Process and assign tasks from the queue, then transport them to their assigned services. + """ + try: + self.task_mgt_strategy.process_tasks( + self.service_registry.get_services, self.transport + ) + logging.info("Tasks processed and transported successfully.") + except Exception as e: + logging.error(f"Error while processing tasks: {e}") + raise ValueError(f"Error processing tasks: {e}") + + def distribute_tasks(self, task: Any) -> None: + """ + Distribute tasks using the task strategy (manual or on-demand assignment). + """ + self.task_mgt_strategy.assign_task(task, self.service_registry.get_services) + logging.info( + f"Task '{task.get('task_id', 'unknown')}' distributed to a service." + ) + + # Orchestration method + def orchestrate_agents(self, tasks: List[Any]) -> None: + """ + Orchestrate agents for task distribution and transportation. + """ + self.submit_tasks(tasks) # Add task to the strategy + self.process_tasks() # Process and transport the task + logging.info("Agents orchestrated successfully.") diff --git a/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/RoundRobinStrategy.py b/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/RoundRobinStrategy.py index 53b1bddd6..d04c5a442 100644 --- a/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/RoundRobinStrategy.py +++ b/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/RoundRobinStrategy.py @@ -57,14 +57,17 @@ def get_task(self, task_id: str) -> Dict[str, Any]: else: raise ValueError(f"Task '{task_id}' not found in assignments.") - def process_tasks(self, service_registry: Callable[[], List[str]]) -> None: + def process_tasks(self, service_registry: Callable[[], List[str]], transport: Callable) -> None: """ Process tasks from the task queue and assign them to services. :param service_registry: Callable that returns available services. + :param transport: Callable used to send tasks to assigned services. """ while not self.task_queue.empty(): task = self.task_queue.get() try: self.assign_task(task, service_registry) + assigned_service = self.task_assignments[task["task_id"]] + transport.send(task, assigned_service) except ValueError as e: raise ValueError(f"Error assigning task: {e}") diff --git a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py index 327f97e34..8ce020ae0 100644 --- a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py +++ b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py @@ -1,83 +1,166 @@ import pytest -from unittest.mock import MagicMock, patch -from swarmauri.control_panels.concrete.ControlPanel import ControlPanel +from unittest.mock import MagicMock +from swarmauri.control_panels.base.ControlPanelBase import ControlPanelBase +from swarmauri.factories.base.FactoryBase import FactoryBase +from swarmauri.service_registries.base.ServiceRegistryBase import ServiceRegistryBase +from swarmauri.task_mgt_strategies.base.TaskMgtStrategyBase import TaskMgtStrategyBase +from swarmauri.transports.base.TransportBase import TransportBase @pytest.fixture def control_panel(): - agent_factory = MagicMock() - service_registry = MagicMock() - task_strategy = MagicMock() - transport = MagicMock() - return ControlPanel(agent_factory, service_registry, task_strategy, transport) + """Fixture to create a ControlPanelBase instance with mocked dependencies.""" + agent_factory = MagicMock(spec=FactoryBase) + service_registry = MagicMock(spec=ServiceRegistryBase) + task_mgt_strategy = MagicMock(spec=TaskMgtStrategyBase) + transport = MagicMock(spec=TransportBase) + + return ControlPanelBase( + agent_factory=agent_factory, + service_registry=service_registry, + task_mgt_strategy=task_mgt_strategy, + transport=transport, + ) def test_create_agent(control_panel): - control_panel.agent_factory.create_agent.return_value = "agent1" - name = "agent1" - role = "role1" - - result = control_panel.create_agent(name, role) - - control_panel.agent_factory.create_agent.assert_called_with(name, role) - control_panel.service_registry.register_service.assert_called_with( - name, {"role": role, "status": "active"} - ) - assert result == "agent1" + """Test the create_agent method.""" + agent_name = "agent1" + agent_role = "worker" + agent = MagicMock() + # Configure mocks + control_panel.agent_factory.create_agent.return_value = agent -def test_distribute_tasks(control_panel): - task = "task1" + # Call the method + result = control_panel.create_agent(agent_name, agent_role) - control_panel.distribute_tasks(task) - - control_panel.task_strategy.assign_task.assert_called_with( - task, control_panel.agent_factory, control_panel.service_registry + # Assertions + control_panel.agent_factory.create_agent.assert_called_once_with( + agent_name, agent_role ) + control_panel.service_registry.register_service.assert_called_once_with( + agent_name, {"role": agent_role, "status": "active"} + ) + assert result == agent -def test_orchestrate_agents(control_panel): - task = "task1" - - control_panel.orchestrate_agents(task) - - with patch.object(control_panel, "manage_agents") as mock_manage_agents: - mock_manage_agents.assert_called_once() - control_panel.distribute_tasks.assert_called_with(task) - - -def test_remove_agent_success(control_panel): - # Arrange - name = "agent1" +def test_remove_agent(control_panel): + """Test the remove_agent method.""" + agent_name = "agent1" agent = MagicMock() + + # Configure mocks control_panel.agent_factory.get_agent_by_name.return_value = agent - control_panel.remove_agent(name) + # Call the method + control_panel.remove_agent(agent_name) - control_panel.service_registry.unregister_service.assert_called_with(name) - control_panel.agent_factory.delete_agent.assert_called_with(name) + # Assertions + control_panel.agent_factory.get_agent_by_name.assert_called_once_with(agent_name) + control_panel.service_registry.unregister_service.assert_called_once_with( + agent_name + ) + control_panel.agent_factory.delete_agent.assert_called_once_with(agent_name) def test_remove_agent_not_found(control_panel): - # Arrange - name = "agent1" + """Test remove_agent when the agent is not found.""" + agent_name = "agent1" + + # Configure mocks control_panel.agent_factory.get_agent_by_name.return_value = None - # Act & Assert - with pytest.raises(ValueError) as excinfo: - control_panel.remove_agent(name) - assert str(excinfo.value) == f"Agent {name} not found." + # Call the method and expect a ValueError + with pytest.raises(ValueError) as exc_info: + control_panel.remove_agent(agent_name) + assert str(exc_info.value) == f"Agent '{agent_name}' not found." def test_list_active_agents(control_panel): - # Arrange + """Test the list_active_agents method.""" agent1 = MagicMock() agent1.name = "agent1" agent2 = MagicMock() agent2.name = "agent2" - control_panel.agent_factory.get_agents.return_value = [agent1, agent2] + agents = [agent1, agent2] + # Configure mocks + control_panel.agent_factory.get_agents.return_value = agents + + # Call the method result = control_panel.list_active_agents() + # Assertions control_panel.agent_factory.get_agents.assert_called_once() assert result == ["agent1", "agent2"] + + +def test_submit_tasks(control_panel): + """Test the submit_tasks method.""" + task1 = {"task_id": "task1"} + task2 = {"task_id": "task2"} + tasks = [task1, task2] + + # Call the method + control_panel.submit_tasks(tasks) + + # Assertions + calls = [((task1,),), ((task2,),)] + control_panel.task_mgt_strategy.add_task.assert_has_calls(calls) + assert control_panel.task_mgt_strategy.add_task.call_count == 2 + + +def test_process_tasks(control_panel): + """Test the process_tasks method.""" + # Call the method + control_panel.process_tasks() + + # Assertions + control_panel.task_mgt_strategy.process_tasks.assert_called_once_with( + control_panel.service_registry.get_services, control_panel.transport + ) + + +def test_process_tasks_exception(control_panel, caplog): + """Test process_tasks when an exception occurs.""" + # Configure mocks + control_panel.task_mgt_strategy.process_tasks.side_effect = Exception("Test error") + + # Call the method + control_panel.process_tasks() + + # Assertions + control_panel.task_mgt_strategy.process_tasks.assert_called_once_with( + control_panel.service_registry.get_services, control_panel.transport + ) + assert "Error while processing tasks: Test error" in caplog.text + + +def test_distribute_tasks(control_panel): + """Test the distribute_tasks method.""" + task = {"task_id": "task1"} + + # Call the method + control_panel.distribute_tasks(task) + + # Assertions + control_panel.task_mgt_strategy.assign_task.assert_called_once_with( + task, control_panel.service_registry.get_services + ) + + +def test_orchestrate_agents(control_panel): + """Test the orchestrate_agents method.""" + tasks = [{"task_id": "task1"}, {"task_id": "task2"}] + + # Configure mocks + control_panel.submit_tasks = MagicMock() + control_panel.process_tasks = MagicMock() + + # Call the method + control_panel.orchestrate_agents(tasks) + + # Assertions + control_panel.submit_tasks.assert_called_once_with(tasks) + control_panel.process_tasks.assert_called_once() diff --git a/pkgs/swarmauri/tests/unit/task_mgt_strategies/RoundRobinStrategy_unit_test.py b/pkgs/swarmauri/tests/unit/task_mgt_strategies/RoundRobinStrategy_unit_test.py index d28d9ab3c..c68a1a7d9 100644 --- a/pkgs/swarmauri/tests/unit/task_mgt_strategies/RoundRobinStrategy_unit_test.py +++ b/pkgs/swarmauri/tests/unit/task_mgt_strategies/RoundRobinStrategy_unit_test.py @@ -1,6 +1,6 @@ import pytest -from swarmauri.task_mgt_strategies.concrete.RoundRobinStrategy import RoundRobinStrategy from unittest.mock import MagicMock +from swarmauri.task_mgt_strategies.concrete.RoundRobinStrategy import RoundRobinStrategy @pytest.fixture @@ -14,8 +14,10 @@ def test_assign_task(round_robin_strategy): task = {"task_id": "task1", "payload": "data"} service_registry = MagicMock(return_value=["service1", "service2"]) + # Execute round_robin_strategy.assign_task(task, service_registry) + # Verify assert round_robin_strategy.task_assignments["task1"] == "service1" assert round_robin_strategy.current_index == 1 @@ -32,51 +34,67 @@ def test_assign_task_no_services(round_robin_strategy): def test_add_task(round_robin_strategy): + # Setup task = {"task_id": "task1", "payload": "data"} + # Execute round_robin_strategy.add_task(task) + # Verify assert not round_robin_strategy.task_queue.empty() queued_task = round_robin_strategy.task_queue.get() assert queued_task == task def test_remove_task(round_robin_strategy): + # Setup task_id = "task1" round_robin_strategy.task_assignments[task_id] = "service1" + # Execute round_robin_strategy.remove_task(task_id) + # Verify assert task_id not in round_robin_strategy.task_assignments def test_remove_task_not_found(round_robin_strategy): + # Setup task_id = "task1" + # Execute & Verify with pytest.raises(ValueError) as exc_info: round_robin_strategy.remove_task(task_id) assert str(exc_info.value) == "Task 'task1' not found in assignments." def test_get_task(round_robin_strategy): + # Setup task_id = "task1" round_robin_strategy.task_assignments[task_id] = "service1" + # Execute result = round_robin_strategy.get_task(task_id) - assert result == {"task_id": task_id, "assigned_service": "service1"} + # Verify + expected_result = {"task_id": task_id, "assigned_service": "service1"} + assert result == expected_result def test_get_task_not_found(round_robin_strategy): + # Setup task_id = "task1" + # Execute & Verify with pytest.raises(ValueError) as exc_info: round_robin_strategy.get_task(task_id) assert str(exc_info.value) == "Task 'task1' not found in assignments." def test_process_tasks(round_robin_strategy): + # Setup service_registry = MagicMock(return_value=["service1", "service2"]) + transport = MagicMock() tasks = [ {"task_id": "task1", "payload": "data1"}, {"task_id": "task2", "payload": "data2"}, @@ -85,9 +103,30 @@ def test_process_tasks(round_robin_strategy): for task in tasks: round_robin_strategy.add_task(task) - round_robin_strategy.process_tasks(service_registry) + # Execute + round_robin_strategy.process_tasks(service_registry, transport) + # Verify assignments assert round_robin_strategy.task_assignments["task1"] == "service1" assert round_robin_strategy.task_assignments["task2"] == "service2" assert round_robin_strategy.task_assignments["task3"] == "service1" assert round_robin_strategy.current_index == 3 + + # Verify that transport.send was called correctly + transport.send.assert_any_call(tasks[0], "service1") + transport.send.assert_any_call(tasks[1], "service2") + transport.send.assert_any_call(tasks[2], "service1") + assert transport.send.call_count == 3 + + +def test_process_tasks_no_services(round_robin_strategy): + # Setup + service_registry = MagicMock(return_value=[]) + transport = MagicMock() + task = {"task_id": "task1", "payload": "data"} + round_robin_strategy.add_task(task) + + # Execute & Verify + with pytest.raises(ValueError) as exc_info: + round_robin_strategy.process_tasks(service_registry, transport) + assert "No services available for task assignment." in str(exc_info.value) From 3a0b00d5e4bde12dfd8be82f704ca24d47e0714b Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Mon, 16 Dec 2024 13:57:42 +0100 Subject: [PATCH 18/95] feat: implement lazy loading for task management strategies and control panels --- .../swarmauri/control_panels/concrete/__init__.py | 13 +++++++++++++ .../task_mgt_strategies/concrete/__init__.py | 13 +++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 pkgs/swarmauri/swarmauri/control_panels/concrete/__init__.py diff --git a/pkgs/swarmauri/swarmauri/control_panels/concrete/__init__.py b/pkgs/swarmauri/swarmauri/control_panels/concrete/__init__.py new file mode 100644 index 000000000..767127c45 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/control_panels/concrete/__init__.py @@ -0,0 +1,13 @@ +from swarmauri.utils._lazy_import import _lazy_import + +# List of control_panels names (file names without the ".py" extension) and corresponding class names +control_panels_files = [ + ("swarmauri.control_panels.concrete.ControlPanel", "ControlPanel"), +] + +# Lazy loading of task_mgt_strategies classes, storing them in variables +for module_name, class_name in control_panels_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded state classes to __all__ +__all__ = [class_name for _, class_name in control_panels_files] diff --git a/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/__init__.py b/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/__init__.py index e69de29bb..3ee5bec23 100644 --- a/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/task_mgt_strategies/concrete/__init__.py @@ -0,0 +1,13 @@ +from swarmauri.utils._lazy_import import _lazy_import + +# List of task_mgt_strategies names (file names without the ".py" extension) and corresponding class names +task_mgt_strategies_files = [ + ("swarmauri.task_mgt_strategies.concrete.RoundRobinStrategy", "RoundRobinStrategy"), +] + +# Lazy loading of task_mgt_strategies classes, storing them in variables +for module_name, class_name in task_mgt_strategies_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded state classes to __all__ +__all__ = [class_name for _, class_name in task_mgt_strategies_files] From 0ce4030da667934a95b56c51180d60365dd5620f Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Mon, 16 Dec 2024 15:37:13 +0100 Subject: [PATCH 19/95] feat: extend ControlPanelBase to inherit from ComponentBase --- .../swarmauri/control_panels/base/ControlPanelBase.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py b/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py index 41653005c..2c7d443a7 100644 --- a/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py +++ b/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py @@ -1,3 +1,4 @@ +from swarmauri_core import ComponentBase from swarmauri_core.control_panels.IControlPanel import IControlPlane from typing import Any, List, Literal from pydantic import Field, ConfigDict @@ -10,7 +11,7 @@ import logging -class ControlPanelBase(IControlPlane): +class ControlPanelBase(IControlPlane, ComponentBase): """ Implementation of the ControlPlane abstract class. This class orchestrates agents, manages tasks, and ensures task distribution From 2d43f1216ed3d86b104667fd17135e3b54e5c0b5 Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Mon, 16 Dec 2024 16:52:31 +0100 Subject: [PATCH 20/95] swarm - feat: add new interfaces and restructure swarm components --- .../swarms(deprecated)/ISwarm.py | 67 +++++++++++++ .../ISwarmAgentRegistration.py | 0 .../ISwarmChainCRUD.py | 0 .../ISwarmComponent.py | 0 .../ISwarmConfigurationExporter.py | 0 .../ISwarmFactory.py | 0 .../swarms(deprecated)/__init__.py | 0 pkgs/core/swarmauri_core/swarms/ISwarm.py | 61 ++++++------ .../swarmauri/swarms(deprecated)/__init__.py | 1 + .../base/SwarmComponentBase.py | 0 .../swarms(deprecated)/base/__init__.py | 0 .../concrete/SimpleSwarmFactory.py | 0 .../swarms(deprecated)/concrete/__init__.py | 11 +++ pkgs/swarmauri/swarmauri/swarms/__init__.py | 1 - .../swarmauri/swarms/base/SwarmBase.py | 95 +++++++++++++++++++ .../swarmauri/swarms/concrete/Swarm.py | 0 .../swarmauri/swarms/concrete/__init__.py | 11 --- 17 files changed, 202 insertions(+), 45 deletions(-) create mode 100644 pkgs/core/swarmauri_core/swarms(deprecated)/ISwarm.py rename pkgs/core/swarmauri_core/{swarms => swarms(deprecated)}/ISwarmAgentRegistration.py (100%) rename pkgs/core/swarmauri_core/{swarms => swarms(deprecated)}/ISwarmChainCRUD.py (100%) rename pkgs/core/swarmauri_core/{swarms => swarms(deprecated)}/ISwarmComponent.py (100%) rename pkgs/core/swarmauri_core/{swarms => swarms(deprecated)}/ISwarmConfigurationExporter.py (100%) rename pkgs/core/swarmauri_core/{swarms => swarms(deprecated)}/ISwarmFactory.py (100%) create mode 100644 pkgs/core/swarmauri_core/swarms(deprecated)/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/swarms(deprecated)/__init__.py rename pkgs/swarmauri/swarmauri/{swarms => swarms(deprecated)}/base/SwarmComponentBase.py (100%) create mode 100644 pkgs/swarmauri/swarmauri/swarms(deprecated)/base/__init__.py rename pkgs/swarmauri/swarmauri/{swarms => swarms(deprecated)}/concrete/SimpleSwarmFactory.py (100%) create mode 100644 pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/__init__.py create mode 100644 pkgs/swarmauri/swarmauri/swarms/base/SwarmBase.py create mode 100644 pkgs/swarmauri/swarmauri/swarms/concrete/Swarm.py diff --git a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarm.py b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarm.py new file mode 100644 index 000000000..7d3ecb8d8 --- /dev/null +++ b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarm.py @@ -0,0 +1,67 @@ +from abc import ABC, abstractmethod +from typing import Any, List, Dict +from datetime import datetime +from swarmauri_core.agents.IAgent import IAgent +from swarmauri_core.chains.ICallableChain import ICallableChain + +class ISwarm(ABC): + """ + Interface for a Swarm, representing a collective of agents capable of performing tasks, executing callable chains, and adaptable configurations. + """ + + # Abstract properties and setters + @property + @abstractmethod + def id(self) -> str: + """Unique identifier for the factory instance.""" + pass + + @id.setter + @abstractmethod + def id(self, value: str) -> None: + pass + + @property + @abstractmethod + def name(self) -> str: + pass + + @name.setter + @abstractmethod + def name(self, value: str) -> None: + pass + + @property + @abstractmethod + def type(self) -> str: + pass + + @type.setter + @abstractmethod + def type(self, value: str) -> None: + pass + + @property + @abstractmethod + def date_created(self) -> datetime: + pass + + @property + @abstractmethod + def last_modified(self) -> datetime: + pass + + @last_modified.setter + @abstractmethod + def last_modified(self, value: datetime) -> None: + pass + + def __hash__(self): + """ + The __hash__ method allows objects of this class to be used in sets and as dictionary keys. + __hash__ should return an integer and be defined based on immutable properties. + This is generally implemented directly in concrete classes rather than in the interface, + but it's declared here to indicate that implementing classes must provide it. + """ + pass + diff --git a/pkgs/core/swarmauri_core/swarms/ISwarmAgentRegistration.py b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmAgentRegistration.py similarity index 100% rename from pkgs/core/swarmauri_core/swarms/ISwarmAgentRegistration.py rename to pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmAgentRegistration.py diff --git a/pkgs/core/swarmauri_core/swarms/ISwarmChainCRUD.py b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmChainCRUD.py similarity index 100% rename from pkgs/core/swarmauri_core/swarms/ISwarmChainCRUD.py rename to pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmChainCRUD.py diff --git a/pkgs/core/swarmauri_core/swarms/ISwarmComponent.py b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmComponent.py similarity index 100% rename from pkgs/core/swarmauri_core/swarms/ISwarmComponent.py rename to pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmComponent.py diff --git a/pkgs/core/swarmauri_core/swarms/ISwarmConfigurationExporter.py b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmConfigurationExporter.py similarity index 100% rename from pkgs/core/swarmauri_core/swarms/ISwarmConfigurationExporter.py rename to pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmConfigurationExporter.py diff --git a/pkgs/core/swarmauri_core/swarms/ISwarmFactory.py b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmFactory.py similarity index 100% rename from pkgs/core/swarmauri_core/swarms/ISwarmFactory.py rename to pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmFactory.py diff --git a/pkgs/core/swarmauri_core/swarms(deprecated)/__init__.py b/pkgs/core/swarmauri_core/swarms(deprecated)/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/core/swarmauri_core/swarms/ISwarm.py b/pkgs/core/swarmauri_core/swarms/ISwarm.py index 7d3ecb8d8..12ceba54c 100644 --- a/pkgs/core/swarmauri_core/swarms/ISwarm.py +++ b/pkgs/core/swarmauri_core/swarms/ISwarm.py @@ -1,67 +1,62 @@ from abc import ABC, abstractmethod -from typing import Any, List, Dict -from datetime import datetime -from swarmauri_core.agents.IAgent import IAgent -from swarmauri_core.chains.ICallableChain import ICallableChain +from typing import Any, List, Tuple + class ISwarm(ABC): - """ - Interface for a Swarm, representing a collective of agents capable of performing tasks, executing callable chains, and adaptable configurations. - """ + """Abstract interface for swarm implementations with both sync/async operations.""" - # Abstract properties and setters @property @abstractmethod - def id(self) -> str: - """Unique identifier for the factory instance.""" + def num_agents(self) -> int: + """Number of agents in the swarm.""" pass - @id.setter @abstractmethod - def id(self, value: str) -> None: + def add_agent(self, agent: Any) -> None: + """Add an agent to the swarm.""" pass - @property @abstractmethod - def name(self) -> str: + def remove_agent(self, agent_id: int) -> None: + """Remove an agent from the swarm.""" pass - @name.setter @abstractmethod - def name(self, value: str) -> None: + def replace_agent(self, agent_id: int, new_agent: Any) -> None: + """Replace an agent in the swarm.""" pass - @property @abstractmethod - def type(self) -> str: + def get_agent_statuses(self) -> List[Tuple[int, Any]]: + """Get status of all agents.""" pass - @type.setter @abstractmethod - def type(self, value: str) -> None: + def distribute_task(self, task: Any) -> None: + """Distribute a task to the swarm.""" pass - @property @abstractmethod - def date_created(self) -> datetime: + def collect_results(self) -> List[Any]: + """Collect results from the swarm.""" pass - @property @abstractmethod - def last_modified(self) -> datetime: + def run(self, tasks: List[Any]) -> List[Any]: + """Execute tasks synchronously.""" pass - @last_modified.setter @abstractmethod - def last_modified(self, value: datetime) -> None: + async def arun(self, tasks: List[Any]) -> List[Any]: + """Execute tasks asynchronously.""" pass - def __hash__(self): - """ - The __hash__ method allows objects of this class to be used in sets and as dictionary keys. - __hash__ should return an integer and be defined based on immutable properties. - This is generally implemented directly in concrete classes rather than in the interface, - but it's declared here to indicate that implementing classes must provide it. - """ + @abstractmethod + def process_task(self, task: Any) -> Any: + """Process a single task synchronously.""" pass + @abstractmethod + async def aprocess_task(self, task: Any) -> Any: + """Process a single task asynchronously.""" + pass diff --git a/pkgs/swarmauri/swarmauri/swarms(deprecated)/__init__.py b/pkgs/swarmauri/swarmauri/swarms(deprecated)/__init__.py new file mode 100644 index 000000000..97c140f08 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/swarms(deprecated)/__init__.py @@ -0,0 +1 @@ +from swarmauri.swarms.concrete import * diff --git a/pkgs/swarmauri/swarmauri/swarms/base/SwarmComponentBase.py b/pkgs/swarmauri/swarmauri/swarms(deprecated)/base/SwarmComponentBase.py similarity index 100% rename from pkgs/swarmauri/swarmauri/swarms/base/SwarmComponentBase.py rename to pkgs/swarmauri/swarmauri/swarms(deprecated)/base/SwarmComponentBase.py diff --git a/pkgs/swarmauri/swarmauri/swarms(deprecated)/base/__init__.py b/pkgs/swarmauri/swarmauri/swarms(deprecated)/base/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/swarms/concrete/SimpleSwarmFactory.py b/pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/SimpleSwarmFactory.py similarity index 100% rename from pkgs/swarmauri/swarmauri/swarms/concrete/SimpleSwarmFactory.py rename to pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/SimpleSwarmFactory.py diff --git a/pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/__init__.py b/pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/__init__.py new file mode 100644 index 000000000..61f84eae6 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/__init__.py @@ -0,0 +1,11 @@ +from swarmauri.utils._lazy_import import _lazy_import + +# List of swarms names (file names without the ".py" extension) and corresponding class names +swarms_files = [("swarmauri.swarms.concrete.SimpleSwarmFactory", "SimpleSwarmFactory")] + +# Lazy loading of swarms classes, storing them in variables +for module_name, class_name in swarms_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded swarms classes to __all__ +__all__ = [class_name for _, class_name in swarms_files] diff --git a/pkgs/swarmauri/swarmauri/swarms/__init__.py b/pkgs/swarmauri/swarmauri/swarms/__init__.py index 97c140f08..e69de29bb 100644 --- a/pkgs/swarmauri/swarmauri/swarms/__init__.py +++ b/pkgs/swarmauri/swarmauri/swarms/__init__.py @@ -1 +0,0 @@ -from swarmauri.swarms.concrete import * diff --git a/pkgs/swarmauri/swarmauri/swarms/base/SwarmBase.py b/pkgs/swarmauri/swarmauri/swarms/base/SwarmBase.py new file mode 100644 index 000000000..8ac192822 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/swarms/base/SwarmBase.py @@ -0,0 +1,95 @@ +import asyncio +from typing import Any, List, Tuple +from swarmauri_core.swarms.ISwarm import ISwarm + + +class SwarmBase(ISwarm): + def __init__( + self, num_agents: int = 5, agent_timeout: float = 1.0, retry_attempts: int = 3 + ): + self._agents: List[Any] = [] + self._num_agents = num_agents + self._agent_timeout = agent_timeout + self._retry_attempts = retry_attempts + self._task_queue: asyncio.Queue = asyncio.Queue() + self._results_queue: asyncio.Queue = asyncio.Queue() + + @property + def num_agents(self) -> int: + return self._num_agents + + def add_agent(self, agent: SwarmBaseAgent) -> None: + self._agents.append(agent) + self._num_agents = len(self._agents) + + def remove_agent(self, agent_id: int) -> None: + if 0 <= agent_id < len(self._agents): + self._agents.pop(agent_id) + self._num_agents = len(self._agents) + + def replace_agent(self, agent_id: int, new_agent: SwarmBaseAgent) -> None: + if 0 <= agent_id < len(self._agents): + self._agents[agent_id] = new_agent + + def get_agent_statuses(self) -> List[Tuple[int, Any]]: + return [(i, agent.status) for i, agent in enumerate(self._agents)] + + def distribute_task(self, task: Any) -> None: + self._task_queue.put_nowait(task) + + def collect_results(self) -> List[Any]: + results = [] + while not self._results_queue.empty(): + results.append(self._results_queue.get_nowait()) + return results + + def run(self, tasks: List[Any]) -> List[Any]: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + return loop.run_until_complete(self.arun(tasks)) + finally: + loop.close() + + async def arun(self, tasks: List[Any]) -> List[Any]: + for task in tasks: + await self._task_queue.put(task) + + agent_tasks = [ + asyncio.create_task(self._process_worker(agent)) for agent in self._agents + ] + await asyncio.gather(*agent_tasks) + return await self._collect_results_async() + + def process_task(self, task: Any) -> Any: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + return loop.run_until_complete(self.aprocess_task(task)) + finally: + loop.close() + + async def aprocess_task(self, task: Any) -> Any: + if not self._agents: + raise RuntimeError("No agents available to process task") + return await self._agents[0].process(task) + + async def _process_worker(self, agent: SwarmBaseAgent) -> None: + retry_count = 0 + while retry_count < self._retry_attempts: + try: + task = await asyncio.wait_for( + self._task_queue.get(), timeout=self._agent_timeout + ) + result = await agent.process(task) + await self._results_queue.put(result) + self._task_queue.task_done() + break + except Exception: + retry_count += 1 + + async def _collect_results_async(self) -> List[Any]: + results = [] + while not self._results_queue.empty(): + results.append(await self._results_queue.get()) + return results diff --git a/pkgs/swarmauri/swarmauri/swarms/concrete/Swarm.py b/pkgs/swarmauri/swarmauri/swarms/concrete/Swarm.py new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/swarmauri/swarmauri/swarms/concrete/__init__.py b/pkgs/swarmauri/swarmauri/swarms/concrete/__init__.py index 61f84eae6..e69de29bb 100644 --- a/pkgs/swarmauri/swarmauri/swarms/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/swarms/concrete/__init__.py @@ -1,11 +0,0 @@ -from swarmauri.utils._lazy_import import _lazy_import - -# List of swarms names (file names without the ".py" extension) and corresponding class names -swarms_files = [("swarmauri.swarms.concrete.SimpleSwarmFactory", "SimpleSwarmFactory")] - -# Lazy loading of swarms classes, storing them in variables -for module_name, class_name in swarms_files: - globals()[class_name] = _lazy_import(module_name, class_name) - -# Adding the lazy-loaded swarms classes to __all__ -__all__ = [class_name for _, class_name in swarms_files] From f6b28da7f68eaa09570ff816ee1c8a3b4d8ce8ba Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Tue, 17 Dec 2024 14:01:56 +0100 Subject: [PATCH 21/95] feat: implement Swarm class with task execution and error handling, add lazy loading for swarm classes --- pkgs/core/swarmauri_core/swarms/ISwarm.py | 62 ++----- .../swarmauri/swarms/base/SwarmBase.py | 160 +++++++++--------- .../swarmauri/swarms/concrete/Swarm.py | 36 ++++ .../swarmauri/swarms/concrete/__init__.py | 13 ++ .../tests/unit/swarms/Swarm_unit_test.py | 91 ++++++++++ 5 files changed, 239 insertions(+), 123 deletions(-) create mode 100644 pkgs/swarmauri/tests/unit/swarms/Swarm_unit_test.py diff --git a/pkgs/core/swarmauri_core/swarms/ISwarm.py b/pkgs/core/swarmauri_core/swarms/ISwarm.py index 12ceba54c..7f3e0bb15 100644 --- a/pkgs/core/swarmauri_core/swarms/ISwarm.py +++ b/pkgs/core/swarmauri_core/swarms/ISwarm.py @@ -1,62 +1,32 @@ from abc import ABC, abstractmethod -from typing import Any, List, Tuple +from typing import Any, Dict, List, Optional, Union class ISwarm(ABC): - """Abstract interface for swarm implementations with both sync/async operations.""" - - @property - @abstractmethod - def num_agents(self) -> int: - """Number of agents in the swarm.""" - pass - - @abstractmethod - def add_agent(self, agent: Any) -> None: - """Add an agent to the swarm.""" - pass - - @abstractmethod - def remove_agent(self, agent_id: int) -> None: - """Remove an agent from the swarm.""" - pass - - @abstractmethod - def replace_agent(self, agent_id: int, new_agent: Any) -> None: - """Replace an agent in the swarm.""" - pass - - @abstractmethod - def get_agent_statuses(self) -> List[Tuple[int, Any]]: - """Get status of all agents.""" - pass - - @abstractmethod - def distribute_task(self, task: Any) -> None: - """Distribute a task to the swarm.""" - pass - - @abstractmethod - def collect_results(self) -> List[Any]: - """Collect results from the swarm.""" - pass + """Abstract base class for swarm implementations""" @abstractmethod - def run(self, tasks: List[Any]) -> List[Any]: - """Execute tasks synchronously.""" + async def exec( + self, + input_data: Union[str, List[str]], + **kwargs: Dict[str, Any], + ) -> Any: + """Execute swarm tasks with given input""" pass @abstractmethod - async def arun(self, tasks: List[Any]) -> List[Any]: - """Execute tasks asynchronously.""" + def get_swarm_status(self) -> Dict[int, Any]: + """Get status of all agents in the swarm""" pass + @property @abstractmethod - def process_task(self, task: Any) -> Any: - """Process a single task synchronously.""" + def agents(self) -> List[Any]: + """Get list of agents in the swarm""" pass + @property @abstractmethod - async def aprocess_task(self, task: Any) -> Any: - """Process a single task asynchronously.""" + def queue_size(self) -> int: + """Get size of task queue""" pass diff --git a/pkgs/swarmauri/swarmauri/swarms/base/SwarmBase.py b/pkgs/swarmauri/swarmauri/swarms/base/SwarmBase.py index 8ac192822..45972f930 100644 --- a/pkgs/swarmauri/swarmauri/swarms/base/SwarmBase.py +++ b/pkgs/swarmauri/swarmauri/swarms/base/SwarmBase.py @@ -1,95 +1,101 @@ import asyncio -from typing import Any, List, Tuple +from typing import Any, Dict, List, Literal, Optional, Union +from pydantic import ConfigDict, Field +from enum import Enum +from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes from swarmauri_core.swarms.ISwarm import ISwarm -class SwarmBase(ISwarm): - def __init__( - self, num_agents: int = 5, agent_timeout: float = 1.0, retry_attempts: int = 3 - ): - self._agents: List[Any] = [] - self._num_agents = num_agents - self._agent_timeout = agent_timeout - self._retry_attempts = retry_attempts - self._task_queue: asyncio.Queue = asyncio.Queue() - self._results_queue: asyncio.Queue = asyncio.Queue() +class SwarmStatus(Enum): + IDLE = "IDLE" + WORKING = "WORKING" + COMPLETED = "COMPLETED" + FAILED = "FAILED" - @property - def num_agents(self) -> int: - return self._num_agents - def add_agent(self, agent: SwarmBaseAgent) -> None: - self._agents.append(agent) - self._num_agents = len(self._agents) +class SwarmBase(ISwarm, ComponentBase): + """Base class for Swarm implementations""" - def remove_agent(self, agent_id: int) -> None: - if 0 <= agent_id < len(self._agents): - self._agents.pop(agent_id) - self._num_agents = len(self._agents) + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + resource: Optional[str] = Field(default=ResourceTypes.SWARM.value, frozen=True) + type: Literal["SwarmBase"] = "SwarmBase" - def replace_agent(self, agent_id: int, new_agent: SwarmBaseAgent) -> None: - if 0 <= agent_id < len(self._agents): - self._agents[agent_id] = new_agent + num_agents: int = Field(default=5, gt=0, le=100) + agent_timeout: float = Field(default=1.0, gt=0) + max_retries: int = Field(default=3, ge=0) + max_queue_size: int = Field(default=10, gt=0) - def get_agent_statuses(self) -> List[Tuple[int, Any]]: - return [(i, agent.status) for i, agent in enumerate(self._agents)] + _agents: List[Any] = [] + _task_queue: Optional[asyncio.Queue] = None + _status: Dict[int, SwarmStatus] = {} - def distribute_task(self, task: Any) -> None: - self._task_queue.put_nowait(task) + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._task_queue = asyncio.Queue(maxsize=self.max_queue_size) + self._initialize_agents() - def collect_results(self) -> List[Any]: - results = [] - while not self._results_queue.empty(): - results.append(self._results_queue.get_nowait()) - return results + def _initialize_agents(self): + self._agents = [self._create_agent() for _ in range(self.num_agents)] + self._status = {i: SwarmStatus.IDLE for i in range(self.num_agents)} - def run(self, tasks: List[Any]) -> List[Any]: - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - try: - return loop.run_until_complete(self.arun(tasks)) - finally: - loop.close() + def _create_agent(self) -> Any: + """create specific agent types""" + raise NotImplementedError("Agent creation method not implemented") - async def arun(self, tasks: List[Any]) -> List[Any]: - for task in tasks: - await self._task_queue.put(task) + @property + def agents(self) -> List[Any]: + return self._agents + + @property + def queue_size(self) -> int: + return self._task_queue.qsize() - agent_tasks = [ - asyncio.create_task(self._process_worker(agent)) for agent in self._agents - ] - await asyncio.gather(*agent_tasks) - return await self._collect_results_async() - def process_task(self, task: Any) -> Any: - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + def get_swarm_status(self) -> Dict[int, SwarmStatus]: + return self._status + + async def _process_task(self, agent_id: int, task: Any, **kwargs) -> Any: + self._status[agent_id] = SwarmStatus.WORKING try: - return loop.run_until_complete(self.aprocess_task(task)) - finally: - loop.close() - - async def aprocess_task(self, task: Any) -> Any: - if not self._agents: - raise RuntimeError("No agents available to process task") - return await self._agents[0].process(task) - - async def _process_worker(self, agent: SwarmBaseAgent) -> None: - retry_count = 0 - while retry_count < self._retry_attempts: - try: - task = await asyncio.wait_for( - self._task_queue.get(), timeout=self._agent_timeout - ) - result = await agent.process(task) - await self._results_queue.put(result) - self._task_queue.task_done() - break - except Exception: - retry_count += 1 - - async def _collect_results_async(self) -> List[Any]: + for _ in range(self.max_retries): + try: + result = await asyncio.wait_for( + self._execute_task(task, agent_id, **kwargs), timeout=self.agent_timeout + ) + self._status[agent_id] = SwarmStatus.COMPLETED + return result + except asyncio.TimeoutError: + continue + self._status[agent_id] = SwarmStatus.FAILED + return None + except Exception as e: + self._status[agent_id] = SwarmStatus.FAILED + raise e + + async def _execute_task(self, task: Any, agent_id: int) -> Any: + """Override this method to implement specific task execution logic""" + raise NotImplementedError("Task execution method not implemented") + + async def exec( + self, input_data: Union[List[str], Any] = [], **kwargs: Optional[Dict] + ) -> List[Any]: + tasks = input_data if isinstance(input_data, list) else [input_data] + for task in tasks: + await self._task_queue.put(task) + results = [] - while not self._results_queue.empty(): - results.append(await self._results_queue.get()) + while not self._task_queue.empty(): + available_agents = [ + i for i, status in self._status.items() if status == SwarmStatus.IDLE + ] + if not available_agents: + await asyncio.sleep(0.1) + continue + + task = await self._task_queue.get() + agent_id = available_agents[0] + result = await self._process_task(agent_id, task, **kwargs) + if result is not None: + results.append(result) + return results diff --git a/pkgs/swarmauri/swarmauri/swarms/concrete/Swarm.py b/pkgs/swarmauri/swarmauri/swarms/concrete/Swarm.py index e69de29bb..2c4c7fee8 100644 --- a/pkgs/swarmauri/swarmauri/swarms/concrete/Swarm.py +++ b/pkgs/swarmauri/swarmauri/swarms/concrete/Swarm.py @@ -0,0 +1,36 @@ +from typing import Any, Dict, List, Literal, Optional, Type, Union +from pydantic import Field + +from swarmauri.swarms.base.SwarmBase import SwarmBase, SwarmStatus + + +class Swarm(SwarmBase): + """Concrete implementation of SwarmBase for task processing""" + + type: Literal["Swarm"] = "Swarm" + agent_class: Type[Any] = Field(description="Agent class to use for swarm") + task_batch_size: int = Field(default=1, gt=0) + + def _create_agent(self) -> Any: + """Create new agent instance""" + return self.agent_class() + + async def _execute_task(self, task: Any, agent_id: int, **kwargs) -> Dict[str, Any]: + """Execute task using specified agent""" + agent = self._agents[agent_id] + try: + result = await agent.process(task, **kwargs) + return { + "agent_id": agent_id, + "status": SwarmStatus.COMPLETED, + "result": result, + } + except Exception as e: + return {"agent_id": agent_id, "status": SwarmStatus.FAILED, "error": str(e)} + + async def exec( + self, input_data: Union[List[str], Any] = [], **kwargs: Optional[Dict] + ) -> List[Dict[str, Any]]: + """Execute tasks in parallel using available agents""" + results = await super().exec(input_data, **kwargs) + return results diff --git a/pkgs/swarmauri/swarmauri/swarms/concrete/__init__.py b/pkgs/swarmauri/swarmauri/swarms/concrete/__init__.py index e69de29bb..98af9eb31 100644 --- a/pkgs/swarmauri/swarmauri/swarms/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/swarms/concrete/__init__.py @@ -0,0 +1,13 @@ +from swarmauri.utils._lazy_import import _lazy_import + +# List of swarms names (file names without the ".py" extension) and corresponding class names +swarms_files = [ + ("swarmauri.swarms.concrete.Swarm", "Swarm") +] + +# Lazy loading of swarms classes, storing them in variables +for module_name, class_name in swarms_files: + globals()[class_name] = _lazy_import(module_name, class_name) + +# Adding the lazy-loaded swarms classes to __all__ +__all__ = [class_name for _, class_name in swarms_files] diff --git a/pkgs/swarmauri/tests/unit/swarms/Swarm_unit_test.py b/pkgs/swarmauri/tests/unit/swarms/Swarm_unit_test.py new file mode 100644 index 000000000..966ee48c9 --- /dev/null +++ b/pkgs/swarmauri/tests/unit/swarms/Swarm_unit_test.py @@ -0,0 +1,91 @@ +from typing import Any +from pydantic import BaseModel +import pytest +import asyncio +from swarmauri.swarms.base.SwarmBase import SwarmStatus +from swarmauri.swarms.concrete.Swarm import Swarm + + +class MockAgent(BaseModel): + async def process(self, task: Any, **kwargs) -> str: + if task == "fail": + raise Exception("Task failed") + return f"Processed {task}" + +@pytest.fixture +def swarm(): + return Swarm(agent_class=MockAgent, num_agents=3, agent_timeout=0.5, max_retries=2) + +@pytest.mark.unit +def test_ubc_resource(swarm): + assert swarm.resource == "Swarm" + + +@pytest.mark.unit +def test_ubc_type(swarm): + assert swarm.type == "Swarm" + + +@pytest.mark.unit +def test_serialization(swarm): + assert swarm.id == Swarm.model_validate_json(swarm.model_dump_json()).id + + +@pytest.mark.asyncio +async def test_swarm_initialization(swarm): + assert len(swarm.agents) == 3 + assert swarm.queue_size == 0 + assert all(s == SwarmStatus.IDLE for s in swarm.get_swarm_status().values()) + + +@pytest.mark.asyncio +async def test_single_task_execution(swarm): + results = await swarm.exec("task1") + assert len(results) == 1 + assert results[0]["status"] == SwarmStatus.COMPLETED + assert results[0]["result"] == "Processed task1" + assert "agent_id" in results[0] + + +@pytest.mark.asyncio +async def test_multiple_tasks_execution(swarm): + tasks = ["task1", "task2", "task3"] + results = await swarm.exec(tasks) + assert len(results) == 3 + assert all(r["status"] == SwarmStatus.COMPLETED for r in results) + assert all("Processed" in r["result"] for r in results) + + +@pytest.mark.asyncio +async def test_failed_task_handling(swarm): + results = await swarm.exec("fail") + assert len(results) == 1 + assert results[0]["status"] == SwarmStatus.FAILED + assert "error" in results[0] + + +@pytest.mark.asyncio(loop_scope="session") +async def test_swarm_status_changes(swarm): + # Create tasks + tasks = ["task1"] * 3 + + # Start execution + task_future = asyncio.create_task(swarm.exec(tasks)) + + # Wait briefly for tasks to start + await asyncio.sleep(0.1) + + # Check intermediate status + status = swarm.get_swarm_status() + assert any( + s in [SwarmStatus.WORKING, SwarmStatus.COMPLETED] for s in status.values() + ) + + # Wait for completion with timeout + try: + results = await asyncio.wait_for(task_future, timeout=2.0) + assert len(results) == len(tasks) + assert all(r["status"] == SwarmStatus.COMPLETED for r in results) + except asyncio.TimeoutError: + task_future.cancel() + raise From 6642ac155befdf2e1da70f04cdb0261d2fa0db56 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 07:13:41 -0600 Subject: [PATCH 22/95] Delete pkgs/core/swarmauri_core/swarms(deprecated)/ISwarm.py --- .../swarms(deprecated)/ISwarm.py | 67 ------------------- 1 file changed, 67 deletions(-) delete mode 100644 pkgs/core/swarmauri_core/swarms(deprecated)/ISwarm.py diff --git a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarm.py b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarm.py deleted file mode 100644 index 7d3ecb8d8..000000000 --- a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarm.py +++ /dev/null @@ -1,67 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Any, List, Dict -from datetime import datetime -from swarmauri_core.agents.IAgent import IAgent -from swarmauri_core.chains.ICallableChain import ICallableChain - -class ISwarm(ABC): - """ - Interface for a Swarm, representing a collective of agents capable of performing tasks, executing callable chains, and adaptable configurations. - """ - - # Abstract properties and setters - @property - @abstractmethod - def id(self) -> str: - """Unique identifier for the factory instance.""" - pass - - @id.setter - @abstractmethod - def id(self, value: str) -> None: - pass - - @property - @abstractmethod - def name(self) -> str: - pass - - @name.setter - @abstractmethod - def name(self, value: str) -> None: - pass - - @property - @abstractmethod - def type(self) -> str: - pass - - @type.setter - @abstractmethod - def type(self, value: str) -> None: - pass - - @property - @abstractmethod - def date_created(self) -> datetime: - pass - - @property - @abstractmethod - def last_modified(self) -> datetime: - pass - - @last_modified.setter - @abstractmethod - def last_modified(self, value: datetime) -> None: - pass - - def __hash__(self): - """ - The __hash__ method allows objects of this class to be used in sets and as dictionary keys. - __hash__ should return an integer and be defined based on immutable properties. - This is generally implemented directly in concrete classes rather than in the interface, - but it's declared here to indicate that implementing classes must provide it. - """ - pass - From 5d3979727362fd1a55e4f6d8358499b5f3969afe Mon Sep 17 00:00:00 2001 From: MichaelDecent Date: Tue, 17 Dec 2024 14:49:45 +0100 Subject: [PATCH 23/95] refactor: remove deprecated swarm components and interfaces --- .../ISwarmAgentRegistration.py | 73 ----------- .../swarms(deprecated)/ISwarmChainCRUD.py | 62 --------- .../swarms(deprecated)/ISwarmComponent.py | 13 -- .../ISwarmConfigurationExporter.py | 33 ----- .../swarms(deprecated)/ISwarmFactory.py | 120 ------------------ .../swarms(deprecated)/__init__.py | 0 .../swarmauri/swarms(deprecated)/__init__.py | 1 - .../base/SwarmComponentBase.py | 15 --- .../swarms(deprecated)/base/__init__.py | 0 .../concrete/SimpleSwarmFactory.py | 50 -------- .../swarms(deprecated)/concrete/__init__.py | 11 -- .../swarmauri/swarms/base/SwarmBase.py | 4 +- 12 files changed, 2 insertions(+), 380 deletions(-) delete mode 100644 pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmAgentRegistration.py delete mode 100644 pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmChainCRUD.py delete mode 100644 pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmComponent.py delete mode 100644 pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmConfigurationExporter.py delete mode 100644 pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmFactory.py delete mode 100644 pkgs/core/swarmauri_core/swarms(deprecated)/__init__.py delete mode 100644 pkgs/swarmauri/swarmauri/swarms(deprecated)/__init__.py delete mode 100644 pkgs/swarmauri/swarmauri/swarms(deprecated)/base/SwarmComponentBase.py delete mode 100644 pkgs/swarmauri/swarmauri/swarms(deprecated)/base/__init__.py delete mode 100644 pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/SimpleSwarmFactory.py delete mode 100644 pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/__init__.py diff --git a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmAgentRegistration.py b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmAgentRegistration.py deleted file mode 100644 index a32dc235d..000000000 --- a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmAgentRegistration.py +++ /dev/null @@ -1,73 +0,0 @@ -from abc import ABC, abstractmethod -from typing import List, Dict, Optional -from swarmauri_core.agents.IAgent import IAgent - -class ISwarmAgentRegistration(ABC): - """ - Interface for registering agents with the swarm, designed to support CRUD operations on IAgent instances. - """ - - @id.setter - @abstractmethod - def registry(self, value: str) -> None: - pass - - @property - @abstractmethod - def registry(self) -> List[IAgent]: - pass - - @abstractmethod - def register_agent(self, agent: IAgent) -> bool: - """ - Register a new agent with the swarm. - - Parameters: - agent (IAgent): An instance of IAgent representing the agent to register. - - Returns: - bool: True if the registration succeeded; False otherwise. - """ - pass - - @abstractmethod - def update_agent(self, agent_id: str, updated_agent: IAgent) -> bool: - """ - Update the details of an existing agent. This could include changing the agent's configuration, - task assignment, or any other mutable attribute. - - Parameters: - agent_id (str): The unique identifier for the agent. - updated_agent (IAgent): An updated IAgent instance to replace the existing one. - - Returns: - bool: True if the update was successful; False otherwise. - """ - pass - - @abstractmethod - def remove_agent(self, agent_id: str) -> bool: - """ - Remove an agent from the swarm based on its unique identifier. - - Parameters: - agent_id (str): The unique identifier for the agent to be removed. - - Returns: - bool: True if the removal was successful; False otherwise. - """ - pass - - @abstractmethod - def get_agent(self, agent_id: str) -> Optional[IAgent]: - """ - Retrieve an agent's instance from its unique identifier. - - Parameters: - agent_id (str): The unique identifier for the agent of interest. - - Returns: - Optional[IAgent]: The IAgent instance if found; None otherwise. - """ - pass - diff --git a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmChainCRUD.py b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmChainCRUD.py deleted file mode 100644 index dcb0cf191..000000000 --- a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmChainCRUD.py +++ /dev/null @@ -1,62 +0,0 @@ -from abc import ABC, abstractmethod -from typing import List, Dict, Any - -class ISwarmChainCRUD(ABC): - """ - Interface to provide CRUD operations for ICallableChain within swarms. - """ - - @abstractmethod - def create_chain(self, chain_id: str, chain_definition: Dict[str, Any]) -> None: - """ - Creates a callable chain with the provided definition. - - Parameters: - - chain_id (str): A unique identifier for the callable chain. - - chain_definition (Dict[str, Any]): The definition of the callable chain including steps and their configurations. - """ - pass - - @abstractmethod - def read_chain(self, chain_id: str) -> Dict[str, Any]: - """ - Retrieves the definition of a callable chain by its identifier. - - Parameters: - - chain_id (str): The unique identifier of the callable chain to be retrieved. - - Returns: - - Dict[str, Any]: The definition of the callable chain. - """ - pass - - @abstractmethod - def update_chain(self, chain_id: str, new_definition: Dict[str, Any]) -> None: - """ - Updates an existing callable chain with a new definition. - - Parameters: - - chain_id (str): The unique identifier of the callable chain to be updated. - - new_definition (Dict[str, Any]): The new definition of the callable chain including updated steps and configurations. - """ - pass - - @abstractmethod - def delete_chain(self, chain_id: str) -> None: - """ - Removes a callable chain from the swarm. - - Parameters: - - chain_id (str): The unique identifier of the callable chain to be removed. - """ - pass - - @abstractmethod - def list_chains(self) -> List[Dict[str, Any]]: - """ - Lists all callable chains currently managed by the swarm. - - Returns: - - List[Dict[str, Any]]: A list of callable chain definitions. - """ - pass \ No newline at end of file diff --git a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmComponent.py b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmComponent.py deleted file mode 100644 index aee78b027..000000000 --- a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmComponent.py +++ /dev/null @@ -1,13 +0,0 @@ -from abc import ABC, abstractmethod - -class ISwarmComponent(ABC): - """ - Interface for defining a general component within a swarm system. - """ - - @abstractmethod - def __init__(self, key: str, name: str): - """ - Initializes a swarm component with a unique key and name. - """ - pass \ No newline at end of file diff --git a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmConfigurationExporter.py b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmConfigurationExporter.py deleted file mode 100644 index 3b1672ca3..000000000 --- a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmConfigurationExporter.py +++ /dev/null @@ -1,33 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Dict -class ISwarmConfigurationExporter(ABC): - - @abstractmethod - def to_dict(self) -> Dict: - """ - Serializes the swarm configuration to a dictionary. - - Returns: - Dict: The serialized configuration as a dictionary. - """ - pass - - @abstractmethod - def to_json(self) -> str: - """ - Serializes the swarm configuration to a JSON string. - - Returns: - str: The serialized configuration as a JSON string. - """ - pass - - @abstractmethod - def to_pickle(self) -> bytes: - """ - Serializes the swarm configuration to a Pickle byte stream. - - Returns: - bytes: The serialized configuration as a Pickle byte stream. - """ - pass \ No newline at end of file diff --git a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmFactory.py b/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmFactory.py deleted file mode 100644 index d26416114..000000000 --- a/pkgs/core/swarmauri_core/swarms(deprecated)/ISwarmFactory.py +++ /dev/null @@ -1,120 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, List, NamedTuple, Optional, Type, Union -from swarmauri_core.swarms.ISwarm import ISwarm -from swarmauri_core.chains.ICallableChain import ICallableChain -from swarmauri_core.agents.IAgent import IAgent - -class Step(NamedTuple): - description: str - callable: Callable # Reference to the function to execute - args: Optional[List[Any]] = None - kwargs: Optional[Dict[str, Any]] = None - -class CallableChainItem(NamedTuple): - key: str # Unique identifier for the item within the chain - execution_context: Dict[str, Any] # Execution context and metadata - steps: List[Step] - -class AgentDefinition(NamedTuple): - type: str - configuration: Dict[str, Any] - capabilities: List[str] - dependencies: List[str] - execution_context: Dict[str, Any] - -class FunctionParameter(NamedTuple): - name: str - type: Type - default: Optional[Any] = None - required: bool = True - -class FunctionDefinition(NamedTuple): - identifier: str - parameters: List[FunctionParameter] - return_type: Type - execution_context: Dict[str, Any] - callable_source: Callable - -class ISwarmFactory(ABC): - - @abstractmethod - def create_swarm(self, *args, **kwargs) -> ISwarm: - """ - Creates and returns a new swarm instance configured with the provided arguments. - """ - pass - - @abstractmethod - def create_agent(self, agent_definition: AgentDefinition) -> IAgent: - """ - Creates a new agent based on the provided enhanced agent definition. - - Args: - agent_definition: An instance of AgentDefinition detailing the agent's setup. - - Returns: - An instance or identifier of the newly created agent. - """ - pass - - @abstractmethod - def create_callable_chain(self, chain_definition: List[CallableChainItem]) -> ICallableChain: - """ - Creates a new callable chain based on the provided definition. - - Args: - chain_definition: Details required to build the chain, such as sequence of functions and arguments. - - Returns: - ICallableChain: The constructed callable chain instance. - """ - pass - - @abstractmethod - def register_function(self, function_definition: FunctionDefinition) -> None: - """ - Registers a function within the factory ecosystem, making it available for callable chains and agents. - - Args: - function_definition: An instance of FunctionDefinition detailing the function's specification. - """ - pass - - @abstractmethod - def export_callable_chains(self, format_type: str = 'json') -> Union[dict, str, bytes]: - """ - Exports configurations of all callable chains in the specified format. - Supported formats: 'json', 'pickle'. - - Args: - format_type (str): The format for exporting the configurations. - - Returns: - Union[dict, str, bytes]: The callable chain configurations in the specified format. - """ - pass - - @abstractmethod - def load_callable_chains(self, chains_data, format_type: str = 'json'): - """ - Loads callable chain configurations from given data. - - Args: - chains_data (Union[dict, str, bytes]): Data containing callable chain configurations. - format_type (str): The format of the provided chains data. - """ - pass - - @abstractmethod - def export_configuration(self, format_type: str = 'json') -> Union[dict, str, bytes]: - """ - Exports the swarm's and agents' configurations in the specified format. - Supported formats: 'json', 'pickle'. Default is 'json'. - - Args: - format_type (str): The format for exporting the configurations. - - Returns: - Union[dict, str, bytes]: The configurations in the specified format. - """ - pass diff --git a/pkgs/core/swarmauri_core/swarms(deprecated)/__init__.py b/pkgs/core/swarmauri_core/swarms(deprecated)/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/pkgs/swarmauri/swarmauri/swarms(deprecated)/__init__.py b/pkgs/swarmauri/swarmauri/swarms(deprecated)/__init__.py deleted file mode 100644 index 97c140f08..000000000 --- a/pkgs/swarmauri/swarmauri/swarms(deprecated)/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from swarmauri.swarms.concrete import * diff --git a/pkgs/swarmauri/swarmauri/swarms(deprecated)/base/SwarmComponentBase.py b/pkgs/swarmauri/swarmauri/swarms(deprecated)/base/SwarmComponentBase.py deleted file mode 100644 index 8e643494a..000000000 --- a/pkgs/swarmauri/swarmauri/swarms(deprecated)/base/SwarmComponentBase.py +++ /dev/null @@ -1,15 +0,0 @@ -from swarmauri_core.swarms.ISwarmComponent import ISwarmComponent - -class SwarmComponentBase(ISwarmComponent): - """ - Interface for defining basics of any component within the swarm system. - """ - def __init__(self, key: str, name: str, superclass: str, module: str, class_name: str, args=None, kwargs=None): - self.key = key - self.name = name - self.superclass = superclass - self.module = module - self.class_name = class_name - self.args = args or [] - self.kwargs = kwargs or {} - \ No newline at end of file diff --git a/pkgs/swarmauri/swarmauri/swarms(deprecated)/base/__init__.py b/pkgs/swarmauri/swarmauri/swarms(deprecated)/base/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/SimpleSwarmFactory.py b/pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/SimpleSwarmFactory.py deleted file mode 100644 index beaec4e49..000000000 --- a/pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/SimpleSwarmFactory.py +++ /dev/null @@ -1,50 +0,0 @@ -import json -import pickle -from typing import List -from swarmauri_core.chains.ISwarmFactory import ( - ISwarmFactory , - CallableChainItem, - AgentDefinition, - FunctionDefinition -) -class SimpleSwarmFactory(ISwarmFactory): - def __init__(self): - self.swarms = [] - self.callable_chains = [] - - def create_swarm(self, agents=[]): - swarm = {"agents": agents} - self.swarms.append(swarm) - return swarm - - def create_agent(self, agent_definition: AgentDefinition): - # For simplicity, agents are stored in a list - # Real-world usage might involve more sophisticated management and instantiation based on type and configuration - agent = {"definition": agent_definition._asdict()} - self.agents.append(agent) - return agent - - def create_callable_chain(self, chain_definition: List[CallableChainItem]): - chain = {"definition": [item._asdict() for item in chain_definition]} - self.callable_chains.append(chain) - return chain - - def register_function(self, function_definition: FunctionDefinition): - if function_definition.identifier in self.functions: - raise ValueError(f"Function {function_definition.identifier} is already registered.") - - self.functions[function_definition.identifier] = function_definition - - def export_configuration(self, format_type: str = 'json'): - # Now exporting both swarms and callable chains - config = {"swarms": self.swarms, "callable_chains": self.callable_chains} - if format_type == "json": - return json.dumps(config) - elif format_type == "pickle": - return pickle.dumps(config) - - def load_configuration(self, config_data, format_type: str = 'json'): - # Loading both swarms and callable chains - config = json.loads(config_data) if format_type == "json" else pickle.loads(config_data) - self.swarms = config.get("swarms", []) - self.callable_chains = config.get("callable_chains", []) \ No newline at end of file diff --git a/pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/__init__.py b/pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/__init__.py deleted file mode 100644 index 61f84eae6..000000000 --- a/pkgs/swarmauri/swarmauri/swarms(deprecated)/concrete/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from swarmauri.utils._lazy_import import _lazy_import - -# List of swarms names (file names without the ".py" extension) and corresponding class names -swarms_files = [("swarmauri.swarms.concrete.SimpleSwarmFactory", "SimpleSwarmFactory")] - -# Lazy loading of swarms classes, storing them in variables -for module_name, class_name in swarms_files: - globals()[class_name] = _lazy_import(module_name, class_name) - -# Adding the lazy-loaded swarms classes to __all__ -__all__ = [class_name for _, class_name in swarms_files] diff --git a/pkgs/swarmauri/swarmauri/swarms/base/SwarmBase.py b/pkgs/swarmauri/swarmauri/swarms/base/SwarmBase.py index 45972f930..88ba5e968 100644 --- a/pkgs/swarmauri/swarmauri/swarms/base/SwarmBase.py +++ b/pkgs/swarmauri/swarmauri/swarms/base/SwarmBase.py @@ -50,7 +50,6 @@ def agents(self) -> List[Any]: def queue_size(self) -> int: return self._task_queue.qsize() - def get_swarm_status(self) -> Dict[int, SwarmStatus]: return self._status @@ -60,7 +59,8 @@ async def _process_task(self, agent_id: int, task: Any, **kwargs) -> Any: for _ in range(self.max_retries): try: result = await asyncio.wait_for( - self._execute_task(task, agent_id, **kwargs), timeout=self.agent_timeout + self._execute_task(task, agent_id, **kwargs), + timeout=self.agent_timeout, ) self._status[agent_id] = SwarmStatus.COMPLETED return result From f38199f81480b3a18456a81a036aaa5cdcf9e92e Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 07:51:19 -0600 Subject: [PATCH 24/95] Update TransportBase.py --- pkgs/swarmauri/swarmauri/transport/base/TransportBase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py b/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py index a25cff190..ddb630e26 100644 --- a/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py +++ b/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py @@ -5,7 +5,7 @@ from swarmauri_core.transport.ITransport import ITransport -class TransportationProtocol(Enum): +class TransportProtocol(Enum): """ Enumeration of transportation protocols supported by the transport layer """ @@ -17,7 +17,7 @@ class TransportationProtocol(Enum): class TransportBase(ITransport, ComponentBase): - allowed_protocols: List[TransportationProtocol] = [] + allowed_protocols: List[TransportProtocol] = [] resource: Optional[str] = Field(default=ResourceTypes.TRANSPORT.value, frozen=True) model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) type: Literal["TransportBase"] = "TransportBase" From c6996722652382ec7a391a5b5dd31bf560441ff2 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 08:01:41 -0600 Subject: [PATCH 25/95] Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index fbe485428..5846c5c09 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -123,7 +123,13 @@ jobs: curl -sSL https://install.python-poetry.org | python3 - echo "PATH=$HOME/.local/bin:$PATH" >> $GITHUB_ENV + - name: Install swarmauri_core package dependency + run: | + cd pkgs/core + poetry install --no-cache --all-extras -vv + - name: Install package dependencies + if: matrix.package_tests.package != 'core' run: | cd pkgs/${{ matrix.package_tests.package }} poetry install --no-cache --all-extras -vv From d27a0b754d849f8c49eaec7a9925f82cfbb08621 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 08:03:12 -0600 Subject: [PATCH 26/95] core - Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index 2270780ce..e913a70a3 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -15,17 +15,25 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.13" + +# Swarmauri swarmauri_core = { path = "./pkgs/core/swarmauri_core" } + +# Dependencies toml = "^0.10.2" httpx = "^0.27.0" joblib = "^1.4.0" numpy = "*" pandas = "*" pydantic = "^2.9.2" -Pillow = ">=8.0,<11.0" typing_extensions = "*" + +# We should remove and only rely on httpx requests = "*" +# This should be set to optional also +Pillow = ">=8.0,<11.0" + # Optional dependencies with versions specified aiofiles = { version = "24.1.0", optional = true } aiohttp = { version = "^3.10.10", optional = true } From 33c0d38cd059f1dc29eb532966190360f0a5b4a4 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 08:04:19 -0600 Subject: [PATCH 27/95] swarm - Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index e913a70a3..e67b513e6 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -14,6 +14,7 @@ classifiers = [ ] [tool.poetry.dependencies] +# Python Version python = ">=3.10,<3.13" # Swarmauri From b52ea84ad73d2d9fc5e1713b0b14de293046a29b Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 08:08:26 -0600 Subject: [PATCH 28/95] Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index e67b513e6..2faf92b76 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -18,7 +18,7 @@ classifiers = [ python = ">=3.10,<3.13" # Swarmauri -swarmauri_core = { path = "./pkgs/core/swarmauri_core" } +swarmauri_core = { path = "./pkgs/core" } # Dependencies toml = "^0.10.2" From 040796268bc8c46f43fe008ef164a5a7dea6f21b Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 08:09:48 -0600 Subject: [PATCH 29/95] Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index 2faf92b76..77d3b1211 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -18,7 +18,7 @@ classifiers = [ python = ">=3.10,<3.13" # Swarmauri -swarmauri_core = { path = "./pkgs/core" } +swarmauri_core = { path = "/pkgs/core" } # Dependencies toml = "^0.10.2" From de08f0b57ce06fe74fb5a084f78a71bdee58a6e3 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 08:13:03 -0600 Subject: [PATCH 30/95] swarm - Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index 77d3b1211..140194615 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -18,7 +18,7 @@ classifiers = [ python = ">=3.10,<3.13" # Swarmauri -swarmauri_core = { path = "/pkgs/core" } +swarmauri_core = { path = "../pkgs/core" } # Dependencies toml = "^0.10.2" From b1b7a5fd573ab8444e262b5a615618584923531d Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 08:15:49 -0600 Subject: [PATCH 31/95] Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index 140194615..dd86ad619 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -18,7 +18,7 @@ classifiers = [ python = ">=3.10,<3.13" # Swarmauri -swarmauri_core = { path = "../pkgs/core" } +swarmauri_core = { path = "../core" } # Dependencies toml = "^0.10.2" From 427e5112aec2ca68f36a86c83f677ec40dde6f04 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:23:38 -0600 Subject: [PATCH 32/95] swarm - Update PubSubTransport.py --- .../swarmauri/swarmauri/transport/concrete/PubSubTransport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py b/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py index 1bf1ea055..f44654791 100644 --- a/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py +++ b/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py @@ -1,11 +1,11 @@ from typing import Dict, Any, List, Optional, Set, Literal from uuid import uuid4 import asyncio -from swarmauri.transport.base.TransportBase import TransportBase, TransportationProtocol +from swarmauri.transport.base.TransportBase import TransportBase, TransportProtocol class PubSubTransport(TransportBase): - allowed_protocols: List[TransportationProtocol] = [TransportationProtocol.PUBSUB] + allowed_protocols: List[TransportProtocol] = [TransportProtocol.PUBSUB] _topics: Dict[str, Set[str]] = {} # Topic to subscriber mappings _subscribers: Dict[str, asyncio.Queue] = {} type: Literal["PubSubTransport"] = "PubSubTransport" From 190fef42c2a6fb88d58ba186f652855656771479 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:27:10 -0600 Subject: [PATCH 33/95] swarm - transport => transports --- .../core/swarmauri_core/{transport => transports}/ITransport.py | 0 pkgs/core/swarmauri_core/{transport => transports}/__init__.py | 0 pkgs/swarmauri/swarmauri/{transport => transports}/__init__.py | 0 .../swarmauri/{transport => transports}/base/TransportBase.py | 2 +- .../swarmauri/{transport => transports}/base/__init__.py | 0 .../{transport => transports}/concrete/PubSubTransport.py | 2 +- .../swarmauri/{transport => transports}/concrete/__init__.py | 0 .../unit/{transport => transports}/PubSubTransport_unit_test.py | 2 +- 8 files changed, 3 insertions(+), 3 deletions(-) rename pkgs/core/swarmauri_core/{transport => transports}/ITransport.py (100%) rename pkgs/core/swarmauri_core/{transport => transports}/__init__.py (100%) rename pkgs/swarmauri/swarmauri/{transport => transports}/__init__.py (100%) rename pkgs/swarmauri/swarmauri/{transport => transports}/base/TransportBase.py (96%) rename pkgs/swarmauri/swarmauri/{transport => transports}/base/__init__.py (100%) rename pkgs/swarmauri/swarmauri/{transport => transports}/concrete/PubSubTransport.py (97%) rename pkgs/swarmauri/swarmauri/{transport => transports}/concrete/__init__.py (100%) rename pkgs/swarmauri/tests/unit/{transport => transports}/PubSubTransport_unit_test.py (98%) diff --git a/pkgs/core/swarmauri_core/transport/ITransport.py b/pkgs/core/swarmauri_core/transports/ITransport.py similarity index 100% rename from pkgs/core/swarmauri_core/transport/ITransport.py rename to pkgs/core/swarmauri_core/transports/ITransport.py diff --git a/pkgs/core/swarmauri_core/transport/__init__.py b/pkgs/core/swarmauri_core/transports/__init__.py similarity index 100% rename from pkgs/core/swarmauri_core/transport/__init__.py rename to pkgs/core/swarmauri_core/transports/__init__.py diff --git a/pkgs/swarmauri/swarmauri/transport/__init__.py b/pkgs/swarmauri/swarmauri/transports/__init__.py similarity index 100% rename from pkgs/swarmauri/swarmauri/transport/__init__.py rename to pkgs/swarmauri/swarmauri/transports/__init__.py diff --git a/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py b/pkgs/swarmauri/swarmauri/transports/base/TransportBase.py similarity index 96% rename from pkgs/swarmauri/swarmauri/transport/base/TransportBase.py rename to pkgs/swarmauri/swarmauri/transports/base/TransportBase.py index ddb630e26..370ed1c8a 100644 --- a/pkgs/swarmauri/swarmauri/transport/base/TransportBase.py +++ b/pkgs/swarmauri/swarmauri/transports/base/TransportBase.py @@ -2,7 +2,7 @@ from pydantic import ConfigDict, Field from enum import Enum, auto from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes -from swarmauri_core.transport.ITransport import ITransport +from swarmauri_core.transports.ITransport import ITransport class TransportProtocol(Enum): diff --git a/pkgs/swarmauri/swarmauri/transport/base/__init__.py b/pkgs/swarmauri/swarmauri/transports/base/__init__.py similarity index 100% rename from pkgs/swarmauri/swarmauri/transport/base/__init__.py rename to pkgs/swarmauri/swarmauri/transports/base/__init__.py diff --git a/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py b/pkgs/swarmauri/swarmauri/transports/concrete/PubSubTransport.py similarity index 97% rename from pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py rename to pkgs/swarmauri/swarmauri/transports/concrete/PubSubTransport.py index f44654791..48303ad6a 100644 --- a/pkgs/swarmauri/swarmauri/transport/concrete/PubSubTransport.py +++ b/pkgs/swarmauri/swarmauri/transports/concrete/PubSubTransport.py @@ -1,7 +1,7 @@ from typing import Dict, Any, List, Optional, Set, Literal from uuid import uuid4 import asyncio -from swarmauri.transport.base.TransportBase import TransportBase, TransportProtocol +from swarmauri.transports.base.TransportBase import TransportBase, TransportProtocol class PubSubTransport(TransportBase): diff --git a/pkgs/swarmauri/swarmauri/transport/concrete/__init__.py b/pkgs/swarmauri/swarmauri/transports/concrete/__init__.py similarity index 100% rename from pkgs/swarmauri/swarmauri/transport/concrete/__init__.py rename to pkgs/swarmauri/swarmauri/transports/concrete/__init__.py diff --git a/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py b/pkgs/swarmauri/tests/unit/transports/PubSubTransport_unit_test.py similarity index 98% rename from pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py rename to pkgs/swarmauri/tests/unit/transports/PubSubTransport_unit_test.py index c8918b086..fb5aeeb5e 100644 --- a/pkgs/swarmauri/tests/unit/transport/PubSubTransport_unit_test.py +++ b/pkgs/swarmauri/tests/unit/transports/PubSubTransport_unit_test.py @@ -2,7 +2,7 @@ import asyncio from uuid import UUID from typing import Any -from swarmauri.transport.concrete.PubSubTransport import ( +from swarmauri.transports.concrete.PubSubTransport import ( PubSubTransport, ) from swarmauri.utils.timeout_wrapper import timeout From 00885b9654859ea8ebc0b1179b386fd1c9120c31 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:29:05 -0600 Subject: [PATCH 34/95] swarm - update transports/concrete/__init__.py --- pkgs/swarmauri/swarmauri/transports/concrete/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/swarmauri/transports/concrete/__init__.py b/pkgs/swarmauri/swarmauri/transports/concrete/__init__.py index 893b547b4..b9f008bb5 100644 --- a/pkgs/swarmauri/swarmauri/transports/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/transports/concrete/__init__.py @@ -2,7 +2,7 @@ # List of transport names (file names without the ".py" extension) and corresponding class names transport_files = [ - ("swarmauri.transport.concrete.PubSubTransport", "PubSubTransport"), + ("swarmauri.transports.concrete.PubSubTransport", "PubSubTransport"), ] # Lazy loading of transport classes, storing them in variables From cb97369bf773b3da2f19d99a8ca517b2fd3b311f Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:31:29 -0600 Subject: [PATCH 35/95] Update __init__.py --- pkgs/swarmauri/swarmauri/transports/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/swarmauri/transports/__init__.py b/pkgs/swarmauri/swarmauri/transports/__init__.py index f3ada0e57..4de373c3c 100644 --- a/pkgs/swarmauri/swarmauri/transports/__init__.py +++ b/pkgs/swarmauri/swarmauri/transports/__init__.py @@ -1 +1 @@ -from swarmauri.transport.concrete import * +from swarmauri.transports.concrete import * From 241b294a5e201a22bfb665cd853d30f81d823f8b Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:39:23 -0600 Subject: [PATCH 36/95] swarm - ControlPanelBase => ControlPanel --- .../tests/unit/control_panels/ControlPanel_unit_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py index 8ce020ae0..be3fa2389 100644 --- a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py +++ b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py @@ -1,6 +1,6 @@ import pytest from unittest.mock import MagicMock -from swarmauri.control_panels.base.ControlPanelBase import ControlPanelBase +from swarmauri.control_panels.concrete.ControlPanel import ControlPanel from swarmauri.factories.base.FactoryBase import FactoryBase from swarmauri.service_registries.base.ServiceRegistryBase import ServiceRegistryBase from swarmauri.task_mgt_strategies.base.TaskMgtStrategyBase import TaskMgtStrategyBase @@ -9,13 +9,13 @@ @pytest.fixture def control_panel(): - """Fixture to create a ControlPanelBase instance with mocked dependencies.""" + """Fixture to create a ControlPanel instance with mocked dependencies.""" agent_factory = MagicMock(spec=FactoryBase) service_registry = MagicMock(spec=ServiceRegistryBase) task_mgt_strategy = MagicMock(spec=TaskMgtStrategyBase) transport = MagicMock(spec=TransportBase) - return ControlPanelBase( + return ControlPanel( agent_factory=agent_factory, service_registry=service_registry, task_mgt_strategy=task_mgt_strategy, From 1b40288a76e53e0dbcc3f71f2eb75c7805d8d54c Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:44:55 -0600 Subject: [PATCH 37/95] swarm - Update ControlPanelBase.py --- .../swarmauri/control_panels/base/ControlPanelBase.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py b/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py index 2c7d443a7..2e36f48cc 100644 --- a/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py +++ b/pkgs/swarmauri/swarmauri/control_panels/base/ControlPanelBase.py @@ -1,8 +1,7 @@ -from swarmauri_core import ComponentBase +from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes from swarmauri_core.control_panels.IControlPanel import IControlPlane from typing import Any, List, Literal from pydantic import Field, ConfigDict -from swarmauri_core.ComponentBase import ResourceTypes from swarmauri.service_registries.base.ServiceRegistryBase import ServiceRegistryBase from swarmauri.factories.base.FactoryBase import FactoryBase from swarmauri.task_mgt_strategies.base.TaskMgtStrategyBase import TaskMgtStrategyBase From c0ad132d2355f1d4bfb90504a7122dab324c1c13 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:48:37 -0600 Subject: [PATCH 38/95] swarm - test serialization of magic mock --- .../control_panels/ControlPanel_unit_test.py | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py index be3fa2389..fa77f283e 100644 --- a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py +++ b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py @@ -6,15 +6,28 @@ from swarmauri.task_mgt_strategies.base.TaskMgtStrategyBase import TaskMgtStrategyBase from swarmauri.transports.base.TransportBase import TransportBase +from unittest.mock import MagicMock +from pydantic import BaseModel + +class SerializableMagicMock(MagicMock, BaseModel): + """A MagicMock class that can be serialized using Pydantic.""" + + def dict(self, *args, **kwargs): + """Serialize the mock object to a dictionary.""" + return {"mock_name": self._mock_name, "calls": self.mock_calls} + + def json(self, *args, **kwargs): + """Serialize the mock object to a JSON string.""" + return super().json(*args, **kwargs) @pytest.fixture def control_panel(): """Fixture to create a ControlPanel instance with mocked dependencies.""" - agent_factory = MagicMock(spec=FactoryBase) - service_registry = MagicMock(spec=ServiceRegistryBase) - task_mgt_strategy = MagicMock(spec=TaskMgtStrategyBase) - transport = MagicMock(spec=TransportBase) - + agent_factory = SerializableMagicMock(spec=FactoryBase) + service_registry = SerializableMagicMock(spec=ServiceRegistryBase) + task_mgt_strategy = SerializableMagicMock(spec=TaskMgtStrategyBase) + transport = SerializableMagicMock(spec=TransportBase) + return ControlPanel( agent_factory=agent_factory, service_registry=service_registry, From 52089295fab5e5f6b806c6e202a9598e711da11f Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:53:22 -0600 Subject: [PATCH 39/95] swarm - test mock --- .../control_panels/ControlPanel_unit_test.py | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py index fa77f283e..93065439d 100644 --- a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py +++ b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py @@ -22,12 +22,28 @@ def json(self, *args, **kwargs): @pytest.fixture def control_panel(): - """Fixture to create a ControlPanel instance with mocked dependencies.""" + """Fixture to create a fully mocked ControlPanel instance with serializable mocks.""" + + # Create serializable mocks for all dependencies agent_factory = SerializableMagicMock(spec=FactoryBase) + agent_factory.create_agent = SerializableMagicMock(return_value="MockAgent") + agent_factory.get_agent_by_name = SerializableMagicMock(return_value="MockAgent") + agent_factory.delete_agent = SerializableMagicMock() + agent_factory.get_agents = SerializableMagicMock(return_value=["MockAgent1", "MockAgent2"]) + service_registry = SerializableMagicMock(spec=ServiceRegistryBase) + service_registry.register_service = SerializableMagicMock() + service_registry.unregister_service = SerializableMagicMock() + service_registry.get_services = SerializableMagicMock(return_value=["service1", "service2"]) + task_mgt_strategy = SerializableMagicMock(spec=TaskMgtStrategyBase) + task_mgt_strategy.add_task = SerializableMagicMock() + task_mgt_strategy.process_tasks = SerializableMagicMock() + task_mgt_strategy.assign_task = SerializableMagicMock() + transport = SerializableMagicMock(spec=TransportBase) - + + # Return the ControlPanel instance with mocked dependencies return ControlPanel( agent_factory=agent_factory, service_registry=service_registry, @@ -35,7 +51,6 @@ def control_panel(): transport=transport, ) - def test_create_agent(control_panel): """Test the create_agent method.""" agent_name = "agent1" From 4bde463231a590c89b33e4dffeff8b6abfea890e Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:54:51 -0600 Subject: [PATCH 40/95] Update ControlPanel_unit_test.py --- .../tests/unit/control_panels/ControlPanel_unit_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py index 93065439d..78d266d46 100644 --- a/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py +++ b/pkgs/swarmauri/tests/unit/control_panels/ControlPanel_unit_test.py @@ -70,7 +70,7 @@ def test_create_agent(control_panel): control_panel.service_registry.register_service.assert_called_once_with( agent_name, {"role": agent_role, "status": "active"} ) - assert result == agent + assert result == "MockAgent" def test_remove_agent(control_panel): From 8b8c77826535c2f11d8e7e4a0b4a45e53cdbd0f9 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:58:53 -0600 Subject: [PATCH 41/95] swarm - fix pipeline inheritance --- pkgs/swarmauri/swarmauri/pipelines/base/PipelineBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/swarmauri/pipelines/base/PipelineBase.py b/pkgs/swarmauri/swarmauri/pipelines/base/PipelineBase.py index da91427b9..6cc302c2b 100644 --- a/pkgs/swarmauri/swarmauri/pipelines/base/PipelineBase.py +++ b/pkgs/swarmauri/swarmauri/pipelines/base/PipelineBase.py @@ -5,7 +5,7 @@ import uuid -class PipelineBase(ComponentBase, IPipeline): +class PipelineBase(IPipeline, ComponentBase): """ Base class providing default behavior for task orchestration, error handling, and result aggregation. From 221666a9271754af7cb69edd0d2c6909a2867038 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:12:55 -0600 Subject: [PATCH 42/95] Create manage_issues.py --- scripts/manage_issues.py | 90 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 scripts/manage_issues.py diff --git a/scripts/manage_issues.py b/scripts/manage_issues.py new file mode 100644 index 000000000..884b8a33f --- /dev/null +++ b/scripts/manage_issues.py @@ -0,0 +1,90 @@ +import os +import json +import requests + +# GitHub API settings +GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") +REPO = os.getenv("REPO") +HEADERS = { + "Authorization": f"Bearer {GITHUB_TOKEN}", + "Accept": "application/vnd.github.v3+json", +} + +ISSUE_LABEL = "pytest-failure" + + +def load_pytest_results(report_file="report.json"): + """Load pytest results from the JSON report.""" + if not os.path.exists(report_file): + print("No pytest report file found.") + return [] + + with open(report_file, "r") as f: + report = json.load(f) + + # Extract failed test cases + failures = [] + for test in report.get("tests", []): + if test["outcome"] == "failed": + failures.append({ + "name": test["name"], + "path": test["nodeid"], + "message": test.get("call", {}).get("longrepr", "No details provided") + }) + return failures + + +def get_existing_issues(): + """Retrieve all existing issues with the pytest-failure label.""" + url = f"https://api.github.com/repos/{REPO}/issues" + params = {"labels": ISSUE_LABEL, "state": "open"} + response = requests.get(url, headers=HEADERS, params=params) + response.raise_for_status() + return response.json() + + +def create_issue(test): + """Create a new GitHub issue for the test failure.""" + url = f"https://api.github.com/repos/{REPO}/issues" + data = { + "title": f"Test Failure: {test['name']}", + "body": f"### Test Case\n```\n{test['path']}\n```\n### Failure Details\n{test['message']}", + "labels": [ISSUE_LABEL] + } + response = requests.post(url, headers=HEADERS, json=data) + response.raise_for_status() + print(f"Issue created for {test['name']}") + + +def add_comment_to_issue(issue_number, test): + """Add a comment to an existing GitHub issue.""" + url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" + data = {"body": f"New failure detected:\n```\n{test['path']}\n```\n### Details\n{test['message']}"} + response = requests.post(url, headers=HEADERS, json=data) + response.raise_for_status() + print(f"Comment added to issue {issue_number} for {test['name']}") + + +def process_failures(): + """Process pytest failures and manage GitHub issues.""" + failures = load_pytest_results() + if not failures: + print("No test failures found.") + return + + existing_issues = get_existing_issues() + + for test in failures: + issue_exists = False + for issue in existing_issues: + if test["name"] in issue["title"]: + add_comment_to_issue(issue["number"], test) + issue_exists = True + break + + if not issue_exists: + create_issue(test) + + +if __name__ == "__main__": + process_failures() From b638af5ecec34d05bb67d5062c4d7b429c6b147a Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:14:28 -0600 Subject: [PATCH 43/95] Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index 5846c5c09..d1369e6a6 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -134,9 +134,18 @@ jobs: cd pkgs/${{ matrix.package_tests.package }} poetry install --no-cache --all-extras -vv - - name: Run all tests for the package + - name: Run tests and save results run: | echo "Running tests for package: ${{ matrix.package_tests.package }}" echo "Test files: ${{ matrix.package_tests.tests }}" cd pkgs/${{ matrix.package_tests.package }} - poetry run pytest ${{ matrix.package_tests.tests }} + poetry run pytest ${{ matrix.package_tests.tests }} --tb=short --json-report --json-report-file=pytest_results.json || true + + - name: Process test results and manage issues + if: always() + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + run: | + python .github/scripts/manage_issues.py + From ee76f4d4e5aeb93dc0cef9e198748c3e3a30607c Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:16:40 -0600 Subject: [PATCH 44/95] Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index d1369e6a6..2d0dbf124 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -147,5 +147,5 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} run: | - python .github/scripts/manage_issues.py + python ../.github/scripts/manage_issues.py From bcb97c5c5b31cb404915eee5884cbfbedf04a784 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:19:59 -0600 Subject: [PATCH 45/95] Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index 2d0dbf124..533e9f79b 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -122,6 +122,7 @@ jobs: run: | curl -sSL https://install.python-poetry.org | python3 - echo "PATH=$HOME/.local/bin:$PATH" >> $GITHUB_ENV + poetry run pip install pytest-json-report - name: Install swarmauri_core package dependency run: | From d7b0f245c3e3c5af3e2477828f6ab70d553e4ec4 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:21:04 -0600 Subject: [PATCH 46/95] swarm - Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index dd86ad619..13924f3de 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -95,10 +95,12 @@ pytest = "^8.0" pytest-asyncio = ">=0.24.0" pytest-timeout = "^2.3.1" pytest-xdist = "^3.6.1" +pytest-json-report = "^1.5.0" python-dotenv = "^1.0.0" jsonschema = "^4.18.5" ipython = "8.28.0" + [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" From be5399a59a02fcae984a0e759ed78cb4d996990d Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:21:19 -0600 Subject: [PATCH 47/95] Update pyproject.toml --- pkgs/core/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/core/pyproject.toml b/pkgs/core/pyproject.toml index 887619499..5558e0cd5 100644 --- a/pkgs/core/pyproject.toml +++ b/pkgs/core/pyproject.toml @@ -25,6 +25,7 @@ flake8 = "^7.0" # Add flake8 as a development dependency pytest = "^8.0" # Ensure pytest is also added if you run tests pytest-asyncio = ">=0.24.0" pytest-xdist = "^3.6.1" +pytest-json-report = "^1.5.0" [build-system] requires = ["poetry-core>=1.0.0"] From a34f8346a360a7d3c1df3143b03f06ba47cf67fc Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:21:37 -0600 Subject: [PATCH 48/95] Update pyproject.toml --- pkgs/community/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/community/pyproject.toml b/pkgs/community/pyproject.toml index db711b327..d8feb1490 100644 --- a/pkgs/community/pyproject.toml +++ b/pkgs/community/pyproject.toml @@ -76,6 +76,7 @@ flake8 = "^7.0" pytest = "^8.0" pytest-asyncio = ">=0.24.0" pytest-xdist = "^3.6.1" +pytest-json-report = "^1.5.0" python-dotenv = "*" [build-system] From 51a86d159ba8a2a18e6b462153071dd005001d97 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:22:17 -0600 Subject: [PATCH 49/95] exp - Update pyproject.toml --- pkgs/experimental/pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/experimental/pyproject.toml b/pkgs/experimental/pyproject.toml index 2bcdb3b7f..a9bef7a00 100644 --- a/pkgs/experimental/pyproject.toml +++ b/pkgs/experimental/pyproject.toml @@ -31,6 +31,8 @@ typing_extensions = "*" flake8 = "^7.0" # Add flake8 as a development dependency pytest = "^8.0" # Ensure pytest is also added if you run tests pytest-asyncio = ">=0.24.0" +pytest-xdist = "^3.6.1" +pytest-json-report = "^1.5.0" [build-system] requires = ["poetry-core>=1.0.0"] From 2a37ef5c222b997df13325ca0badc8b6396437dd Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:23:59 -0600 Subject: [PATCH 50/95] Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index 533e9f79b..2d0dbf124 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -122,7 +122,6 @@ jobs: run: | curl -sSL https://install.python-poetry.org | python3 - echo "PATH=$HOME/.local/bin:$PATH" >> $GITHUB_ENV - poetry run pip install pytest-json-report - name: Install swarmauri_core package dependency run: | From 49be12f05cfb6c32626e98113c538bd5c9770f7d Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:25:35 -0600 Subject: [PATCH 51/95] cicd - Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index 2d0dbf124..23332133d 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -147,5 +147,5 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} run: | - python ../.github/scripts/manage_issues.py + python scripts/manage_issues.py From 4a1d219205d41d401fc4ddc1cd676c458fae9355 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:27:50 -0600 Subject: [PATCH 52/95] swarm - Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index 13924f3de..6177ffd54 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -99,7 +99,7 @@ pytest-json-report = "^1.5.0" python-dotenv = "^1.0.0" jsonschema = "^4.18.5" ipython = "8.28.0" - +requests = "*" [build-system] requires = ["poetry-core>=1.0.0"] From 1d1489d9eb61ada91f39fae4a96ff452f127c834 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:29:07 -0600 Subject: [PATCH 53/95] swarm - Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index 6177ffd54..624a74b0e 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -30,7 +30,7 @@ pydantic = "^2.9.2" typing_extensions = "*" # We should remove and only rely on httpx -requests = "*" +requests = "^2.32.3" # This should be set to optional also Pillow = ">=8.0,<11.0" @@ -98,8 +98,8 @@ pytest-xdist = "^3.6.1" pytest-json-report = "^1.5.0" python-dotenv = "^1.0.0" jsonschema = "^4.18.5" -ipython = "8.28.0" -requests = "*" +ipython = "^8.28.0" +requests = "^2.32.3" [build-system] requires = ["poetry-core>=1.0.0"] From 15cb5276162d28f98ed5cd0fafb23366999651d8 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:33:51 -0600 Subject: [PATCH 54/95] Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index 23332133d..612a77ee4 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -147,5 +147,5 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} run: | - python scripts/manage_issues.py + poetry run python .github/scripts/manage_issues.py From b2a9a6f1303db05e07a6cef4bacbb00fd13dd193 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:36:27 -0600 Subject: [PATCH 55/95] cicd - Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index 612a77ee4..384ee2cd1 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -147,5 +147,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} run: | - poetry run python .github/scripts/manage_issues.py + cd pkgs/${{ matrix.package_tests.package }} + poetry run python scripts/manage_issues.py From 7208542eec28750a6cd7cc689fbf350ead71e31f Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:39:59 -0600 Subject: [PATCH 56/95] cicd - Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index 384ee2cd1..07c1cdf4c 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -147,6 +147,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} run: | - cd pkgs/${{ matrix.package_tests.package }} - poetry run python scripts/manage_issues.py + cd pkgs/swarmauri # Change to the directory containing pyproject.toml + poetry run python ../../scripts/manage_issues.py + From 74317ce83dceac3075ee78b5af17a6ba850516eb Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:42:50 -0600 Subject: [PATCH 57/95] cicd - Update manage_issues.py --- scripts/manage_issues.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/scripts/manage_issues.py b/scripts/manage_issues.py index 884b8a33f..df844009d 100644 --- a/scripts/manage_issues.py +++ b/scripts/manage_issues.py @@ -1,6 +1,7 @@ import os import json import requests +import argparse # GitHub API settings GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") @@ -12,11 +13,21 @@ ISSUE_LABEL = "pytest-failure" - -def load_pytest_results(report_file="report.json"): +def parse_arguments(): + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description="Manage GitHub Issues for Test Failures") + parser.add_argument( + "--results-file", + type=str, + default="pytest_results.json", + help="Path to the pytest JSON report file (default: pytest_results.json)", + ) + return parser.parse_args() + +def load_pytest_results(report_file): """Load pytest results from the JSON report.""" if not os.path.exists(report_file): - print("No pytest report file found.") + print(f"Report file not found: {report_file}") return [] with open(report_file, "r") as f: @@ -33,7 +44,6 @@ def load_pytest_results(report_file="report.json"): }) return failures - def get_existing_issues(): """Retrieve all existing issues with the pytest-failure label.""" url = f"https://api.github.com/repos/{REPO}/issues" @@ -42,7 +52,6 @@ def get_existing_issues(): response.raise_for_status() return response.json() - def create_issue(test): """Create a new GitHub issue for the test failure.""" url = f"https://api.github.com/repos/{REPO}/issues" @@ -55,7 +64,6 @@ def create_issue(test): response.raise_for_status() print(f"Issue created for {test['name']}") - def add_comment_to_issue(issue_number, test): """Add a comment to an existing GitHub issue.""" url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" @@ -64,10 +72,9 @@ def add_comment_to_issue(issue_number, test): response.raise_for_status() print(f"Comment added to issue {issue_number} for {test['name']}") - -def process_failures(): +def process_failures(report_file): """Process pytest failures and manage GitHub issues.""" - failures = load_pytest_results() + failures = load_pytest_results(report_file) if not failures: print("No test failures found.") return @@ -85,6 +92,6 @@ def process_failures(): if not issue_exists: create_issue(test) - if __name__ == "__main__": - process_failures() + args = parse_arguments() + process_failures(args.results_file) From 5732a54f60db13776dd43247114cb8f36aaa0424 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:44:40 -0600 Subject: [PATCH 58/95] Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index 07c1cdf4c..57cfafa92 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -148,6 +148,6 @@ jobs: REPO: ${{ github.repository }} run: | cd pkgs/swarmauri # Change to the directory containing pyproject.toml - poetry run python ../../scripts/manage_issues.py + poetry run python ../../scripts/manage_issues.py --results-file=pytest_results.json From 89a047b5dbb7a49c793361249b41f650cef3f583 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:46:34 -0600 Subject: [PATCH 59/95] cicd - Update manage_issues.py --- scripts/manage_issues.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/manage_issues.py b/scripts/manage_issues.py index df844009d..8ad253e6a 100644 --- a/scripts/manage_issues.py +++ b/scripts/manage_issues.py @@ -36,14 +36,20 @@ def load_pytest_results(report_file): # Extract failed test cases failures = [] for test in report.get("tests", []): - if test["outcome"] == "failed": + if test.get("outcome") == "failed": + # Safely extract keys with defaults + test_name = test.get("nodeid", "Unknown test") + test_path = test.get("nodeid", "Unknown path") + failure_message = test.get("call", {}).get("longrepr", "No failure details available") + failures.append({ - "name": test["name"], - "path": test["nodeid"], - "message": test.get("call", {}).get("longrepr", "No details provided") + "name": test_name, + "path": test_path, + "message": failure_message }) return failures + def get_existing_issues(): """Retrieve all existing issues with the pytest-failure label.""" url = f"https://api.github.com/repos/{REPO}/issues" From c5ad24ffeed5cb39ea874cbbb2259521776fa0d4 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:48:52 -0600 Subject: [PATCH 60/95] cicd - Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index 57cfafa92..c593a21a0 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -99,6 +99,9 @@ jobs: run-tests: needs: detect-changed-files runs-on: ubuntu-latest + permissions: + issues: write + contents: read if: ${{ needs.detect-changed-files.outputs.matrix != '[]' }} strategy: fail-fast: false From 684054d60f3213b6a0a762c791241a248f0ba8e8 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:55:26 -0600 Subject: [PATCH 61/95] cicd - Update manage_issues.py --- scripts/manage_issues.py | 80 ++++++++++++++++++++++++++++++---------- 1 file changed, 60 insertions(+), 20 deletions(-) diff --git a/scripts/manage_issues.py b/scripts/manage_issues.py index 8ad253e6a..b7da50ab2 100644 --- a/scripts/manage_issues.py +++ b/scripts/manage_issues.py @@ -11,7 +11,9 @@ "Accept": "application/vnd.github.v3+json", } -ISSUE_LABEL = "pytest-failure" +BASE_BRANCH = os.getenv("GITHUB_REF", "unknown").split("/")[-1] +COMMIT_SHA = os.getenv("GITHUB_SHA", "unknown") +WORKFLOW_RUN_URL = os.getenv("GITHUB_SERVER_URL", "https://github.com") + f"/{REPO}/actions/runs/{os.getenv('GITHUB_RUN_ID', 'unknown')}" def parse_arguments(): """Parse command-line arguments.""" @@ -22,6 +24,12 @@ def parse_arguments(): default="pytest_results.json", help="Path to the pytest JSON report file (default: pytest_results.json)", ) + parser.add_argument( + "--package", + type=str, + required=True, + help="Name of the matrix package where the error occurred.", + ) return parser.parse_args() def load_pytest_results(report_file): @@ -33,52 +41,84 @@ def load_pytest_results(report_file): with open(report_file, "r") as f: report = json.load(f) - # Extract failed test cases failures = [] for test in report.get("tests", []): if test.get("outcome") == "failed": - # Safely extract keys with defaults test_name = test.get("nodeid", "Unknown test") - test_path = test.get("nodeid", "Unknown path") failure_message = test.get("call", {}).get("longrepr", "No failure details available") - failures.append({ "name": test_name, - "path": test_path, + "path": test.get("nodeid", "Unknown path"), "message": failure_message }) return failures - def get_existing_issues(): """Retrieve all existing issues with the pytest-failure label.""" url = f"https://api.github.com/repos/{REPO}/issues" - params = {"labels": ISSUE_LABEL, "state": "open"} + params = {"labels": "pytest-failure", "state": "open"} response = requests.get(url, headers=HEADERS, params=params) response.raise_for_status() return response.json() -def create_issue(test): +def create_issue(test, package): """Create a new GitHub issue for the test failure.""" url = f"https://api.github.com/repos/{REPO}/issues" + + # Construct the issue body with enhanced documentation data = { - "title": f"Test Failure: {test['name']}", - "body": f"### Test Case\n```\n{test['path']}\n```\n### Failure Details\n{test['message']}", - "labels": [ISSUE_LABEL] + "title": f"[Test Case Failure]: {test['name']}", + "body": f""" +### Test Case: +{test['path']} + +### Failure Details: +{test['message']} + +--- + +### Context: +- **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) +- **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) +- **Workflow Run**: [View Run]({WORKFLOW_RUN_URL}) +- **Matrix Package**: `{package}` + +--- + +### Label: +This issue is auto-labeled for the `{package}` package. +""", + "labels": ["pytest-failure", package] } response = requests.post(url, headers=HEADERS, json=data) response.raise_for_status() - print(f"Issue created for {test['name']}") + print(f"Issue created for {test['name']} in package '{package}'.") -def add_comment_to_issue(issue_number, test): +def add_comment_to_issue(issue_number, test, package): """Add a comment to an existing GitHub issue.""" url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" - data = {"body": f"New failure detected:\n```\n{test['path']}\n```\n### Details\n{test['message']}"} + data = {"body": f""" +New failure detected: + +### Test Case: +{test['path']} + +### Details: +{test['message']} + +--- + +### Context: +- **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) +- **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) +- **Workflow Run**: [View Run]({WORKFLOW_RUN_URL}) +- **Matrix Package**: `{package}` +"""} response = requests.post(url, headers=HEADERS, json=data) response.raise_for_status() - print(f"Comment added to issue {issue_number} for {test['name']}") + print(f"Comment added to issue {issue_number} for {test['name']}.") -def process_failures(report_file): +def process_failures(report_file, package): """Process pytest failures and manage GitHub issues.""" failures = load_pytest_results(report_file) if not failures: @@ -91,13 +131,13 @@ def process_failures(report_file): issue_exists = False for issue in existing_issues: if test["name"] in issue["title"]: - add_comment_to_issue(issue["number"], test) + add_comment_to_issue(issue["number"], test, package) issue_exists = True break if not issue_exists: - create_issue(test) + create_issue(test, package) if __name__ == "__main__": args = parse_arguments() - process_failures(args.results_file) + process_failures(args.results_file, args.package) From 3b0100a37e06d99890dafd503ba287b49e695970 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:56:49 -0600 Subject: [PATCH 62/95] Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index c593a21a0..df733c3db 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -151,6 +151,6 @@ jobs: REPO: ${{ github.repository }} run: | cd pkgs/swarmauri # Change to the directory containing pyproject.toml - poetry run python ../../scripts/manage_issues.py --results-file=pytest_results.json + poetry run python ../../scripts/manage_issues.py --results-file=pytest_results.json --package=${{ matrix.package_tests.package }} From 0c4eb4272a8317b6c6237a4d1f501b1c67a4bfcf Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:05:31 -0600 Subject: [PATCH 63/95] cicd - Update manage_issues.py --- scripts/manage_issues.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/manage_issues.py b/scripts/manage_issues.py index b7da50ab2..d08e03acd 100644 --- a/scripts/manage_issues.py +++ b/scripts/manage_issues.py @@ -11,7 +11,7 @@ "Accept": "application/vnd.github.v3+json", } -BASE_BRANCH = os.getenv("GITHUB_REF", "unknown").split("/")[-1] +BASE_BRANCH = os.getenv("GITHUB_HEAD_REF") or os.getenv("GITHUB_REF", "unknown").split("/")[-1] COMMIT_SHA = os.getenv("GITHUB_SHA", "unknown") WORKFLOW_RUN_URL = os.getenv("GITHUB_SERVER_URL", "https://github.com") + f"/{REPO}/actions/runs/{os.getenv('GITHUB_RUN_ID', 'unknown')}" From 358f6ae930bc66a20c67e1e3c79059ee85fc4719 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:15:48 -0600 Subject: [PATCH 64/95] Update manage_issues.py --- scripts/manage_issues.py | 79 +++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 42 deletions(-) diff --git a/scripts/manage_issues.py b/scripts/manage_issues.py index d08e03acd..89cde4393 100644 --- a/scripts/manage_issues.py +++ b/scripts/manage_issues.py @@ -1,6 +1,8 @@ import os import json import requests +from swarmauri.llms.concrete.GroqModel import GroqModel +from swarmauri.agents.concrete.SimpleConversationAgent import SimpleConversationAgent import argparse # GitHub API settings @@ -11,6 +13,11 @@ "Accept": "application/vnd.github.v3+json", } +# GroqModel Initialization +GROQ_API_KEY = os.getenv("GROQ_API_KEY") +llm = GroqModel(api_key=GROQ_API_KEY) +agent = SimpleConversationAgent(llm=llm) + BASE_BRANCH = os.getenv("GITHUB_HEAD_REF") or os.getenv("GITHUB_REF", "unknown").split("/")[-1] COMMIT_SHA = os.getenv("GITHUB_SHA", "unknown") WORKFLOW_RUN_URL = os.getenv("GITHUB_SERVER_URL", "https://github.com") + f"/{REPO}/actions/runs/{os.getenv('GITHUB_RUN_ID', 'unknown')}" @@ -18,18 +25,8 @@ def parse_arguments(): """Parse command-line arguments.""" parser = argparse.ArgumentParser(description="Manage GitHub Issues for Test Failures") - parser.add_argument( - "--results-file", - type=str, - default="pytest_results.json", - help="Path to the pytest JSON report file (default: pytest_results.json)", - ) - parser.add_argument( - "--package", - type=str, - required=True, - help="Name of the matrix package where the error occurred.", - ) + parser.add_argument("--results-file", type=str, required=True, help="Path to the pytest JSON report file") + parser.add_argument("--package", type=str, required=True, help="Name of the matrix package where the error occurred.") return parser.parse_args() def load_pytest_results(report_file): @@ -53,6 +50,24 @@ def load_pytest_results(report_file): }) return failures +def ask_groq_for_fix(test_name, failure_message, stack_trace): + """Ask GroqModel for suggestions on fixing the test failure.""" + prompt = f""" + I have a failing test case named '{test_name}' in a Python project. The error message is: + {failure_message} + + The stack trace is: + {stack_trace} + + Can you help me identify the cause of this failure and suggest a fix? + """ + try: + response = agent.exec(input_str=prompt) + return response + except Exception as e: + print(f"Error communicating with Groq: {e}") + return "Unable to retrieve suggestions from Groq at this time." + def get_existing_issues(): """Retrieve all existing issues with the pytest-failure label.""" url = f"https://api.github.com/repos/{REPO}/issues" @@ -63,9 +78,10 @@ def get_existing_issues(): def create_issue(test, package): """Create a new GitHub issue for the test failure.""" + groq_suggestion = ask_groq_for_fix(test["name"], test["message"], test["message"]) url = f"https://api.github.com/repos/{REPO}/issues" - # Construct the issue body with enhanced documentation + # Construct the issue body data = { "title": f"[Test Case Failure]: {test['name']}", "body": f""" @@ -77,46 +93,25 @@ def create_issue(test, package): --- +### Suggested Fix (via Groq): +{groq_suggestion} + +--- + ### Context: - **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) - **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) - **Workflow Run**: [View Run]({WORKFLOW_RUN_URL}) - **Matrix Package**: `{package}` ---- - -### Label: +### Labels: This issue is auto-labeled for the `{package}` package. """, "labels": ["pytest-failure", package] } response = requests.post(url, headers=HEADERS, json=data) response.raise_for_status() - print(f"Issue created for {test['name']} in package '{package}'.") - -def add_comment_to_issue(issue_number, test, package): - """Add a comment to an existing GitHub issue.""" - url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" - data = {"body": f""" -New failure detected: - -### Test Case: -{test['path']} - -### Details: -{test['message']} - ---- - -### Context: -- **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) -- **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) -- **Workflow Run**: [View Run]({WORKFLOW_RUN_URL}) -- **Matrix Package**: `{package}` -"""} - response = requests.post(url, headers=HEADERS, json=data) - response.raise_for_status() - print(f"Comment added to issue {issue_number} for {test['name']}.") + print(f"Issue created for {test['name']} with Groq suggestion in package '{package}'.") def process_failures(report_file, package): """Process pytest failures and manage GitHub issues.""" @@ -131,7 +126,7 @@ def process_failures(report_file, package): issue_exists = False for issue in existing_issues: if test["name"] in issue["title"]: - add_comment_to_issue(issue["number"], test, package) + print(f"Issue already exists for {test['name']}. Skipping...") issue_exists = True break From 4b4ec5747c586c8beef8bf68626a1d58b0b7bd69 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:21:23 -0600 Subject: [PATCH 65/95] cicd - Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index df733c3db..f5c7efb92 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -147,6 +147,7 @@ jobs: - name: Process test results and manage issues if: always() env: + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} run: | From 94500037118364ab43f616bda7e6a52d7ca48ae0 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:28:02 -0600 Subject: [PATCH 66/95] Update manage_issues.py --- scripts/manage_issues.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/manage_issues.py b/scripts/manage_issues.py index 89cde4393..ddd177ab2 100644 --- a/scripts/manage_issues.py +++ b/scripts/manage_issues.py @@ -3,6 +3,7 @@ import requests from swarmauri.llms.concrete.GroqModel import GroqModel from swarmauri.agents.concrete.SimpleConversationAgent import SimpleConversationAgent +from swarmauri.conversations.concrete.Conversation import Conversation import argparse # GitHub API settings @@ -16,7 +17,6 @@ # GroqModel Initialization GROQ_API_KEY = os.getenv("GROQ_API_KEY") llm = GroqModel(api_key=GROQ_API_KEY) -agent = SimpleConversationAgent(llm=llm) BASE_BRANCH = os.getenv("GITHUB_HEAD_REF") or os.getenv("GITHUB_REF", "unknown").split("/")[-1] COMMIT_SHA = os.getenv("GITHUB_SHA", "unknown") @@ -62,6 +62,7 @@ def ask_groq_for_fix(test_name, failure_message, stack_trace): Can you help me identify the cause of this failure and suggest a fix? """ try: + agent = SimpleConversationAgent(llm=llm, conversation=Conversation()) response = agent.exec(input_str=prompt) return response except Exception as e: From 8130497e220f0a931747337979f3a1de1b368050 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:32:43 -0600 Subject: [PATCH 67/95] Create LazyLoader.py --- pkgs/swarmauri/swarmauri/utils/LazyLoader.py | 31 ++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 pkgs/swarmauri/swarmauri/utils/LazyLoader.py diff --git a/pkgs/swarmauri/swarmauri/utils/LazyLoader.py b/pkgs/swarmauri/swarmauri/utils/LazyLoader.py new file mode 100644 index 000000000..065065861 --- /dev/null +++ b/pkgs/swarmauri/swarmauri/utils/LazyLoader.py @@ -0,0 +1,31 @@ +import importlib + +class LazyLoader: + def __init__(self, module_name, class_name): + self.module_name = module_name + self.class_name = class_name + self._loaded_class = None + + def _load_class(self): + if self._loaded_class is None: + try: + module = importlib.import_module(self.module_name) + self._loaded_class = getattr(module, self.class_name) + except ImportError: + print( + f"Warning: The module '{self.module_name}' is not available. " + f"Please install the necessary dependencies to enable this functionality." + ) + self._loaded_class = None + except AttributeError: + print( + f"Warning: The class '{self.class_name}' was not found in module '{self.module_name}'." + ) + self._loaded_class = None + return self._loaded_class + + def __getattr__(self, item): + loaded_class = self._load_class() + if loaded_class is None: + raise ImportError(f"Unable to load class {self.class_name} from {self.module_name}") + return getattr(loaded_class, item) From aa915b6fc707304713ce8ec8ba0e1ffdb13e9ebb Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:34:42 -0600 Subject: [PATCH 68/95] swarm - shift from _lazy_import to LazyLoader --- pkgs/swarmauri/swarmauri/tools/concrete/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkgs/swarmauri/swarmauri/tools/concrete/__init__.py b/pkgs/swarmauri/swarmauri/tools/concrete/__init__.py index 5b7d61054..173b3ac4b 100644 --- a/pkgs/swarmauri/swarmauri/tools/concrete/__init__.py +++ b/pkgs/swarmauri/swarmauri/tools/concrete/__init__.py @@ -1,4 +1,4 @@ -from swarmauri.utils._lazy_import import _lazy_import +from swarmauri.utils.LazyLoader import LazyLoader # List of tool names (file names without the ".py" extension) and corresponding class names tool_files = [ @@ -27,9 +27,9 @@ ("swarmauri.tools.concrete.WeatherTool", "WeatherTool"), ] -# Lazy loading of tools, storing them in variables +# Lazy loading of tools using LazyLoader for module_name, class_name in tool_files: - globals()[class_name] = _lazy_import(module_name, class_name) + globals()[class_name] = LazyLoader(module_name, class_name) -# Adding the lazy-loaded tools to __all__ +# Adding tools to __all__ (still safe because LazyLoader doesn't raise errors until accessed) __all__ = [class_name for _, class_name in tool_files] From d15ef7887bfbfe8d33c00b4cfb42c2ef50c7cc89 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:39:18 -0600 Subject: [PATCH 69/95] cicd - Update manage_issues.py --- scripts/manage_issues.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/scripts/manage_issues.py b/scripts/manage_issues.py index ddd177ab2..bab5fb1ad 100644 --- a/scripts/manage_issues.py +++ b/scripts/manage_issues.py @@ -102,6 +102,7 @@ def create_issue(test, package): ### Context: - **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) - **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) +- **Commit Tree**: [{COMMIT_SHA}](https://github.com/{REPO}/tree/{COMMIT_SHA}) - **Workflow Run**: [View Run]({WORKFLOW_RUN_URL}) - **Matrix Package**: `{package}` @@ -114,6 +115,31 @@ def create_issue(test, package): response.raise_for_status() print(f"Issue created for {test['name']} with Groq suggestion in package '{package}'.") +def add_comment_to_issue(issue_number, test, package): + """Add a comment to an existing GitHub issue.""" + url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" + data = {"body": f""" +New failure detected: + +### Test Case: +{test['path']} + +### Details: +{test['message']} + +--- + +### Context: +- **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) +- **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) +- **Commit Tree**: [{COMMIT_SHA}](https://github.com/{REPO}/tree/{COMMIT_SHA}) +- **Workflow Run**: [View Run]({WORKFLOW_RUN_URL}) +- **Matrix Package**: `{package}` +"""} + response = requests.post(url, headers=HEADERS, json=data) + response.raise_for_status() + print(f"Comment added to issue {issue_number} for {test['name']}.") + def process_failures(report_file, package): """Process pytest failures and manage GitHub issues.""" failures = load_pytest_results(report_file) @@ -127,7 +153,7 @@ def process_failures(report_file, package): issue_exists = False for issue in existing_issues: if test["name"] in issue["title"]: - print(f"Issue already exists for {test['name']}. Skipping...") + add_comment_to_issue(issue["number"], test, package) issue_exists = True break From 36817168be4950dcb16df38601e1078457ccf701 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:56:45 -0600 Subject: [PATCH 70/95] Update pyproject.toml --- pkgs/swarmauri/pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkgs/swarmauri/pyproject.toml b/pkgs/swarmauri/pyproject.toml index 624a74b0e..18ea3f273 100644 --- a/pkgs/swarmauri/pyproject.toml +++ b/pkgs/swarmauri/pyproject.toml @@ -49,7 +49,7 @@ yake = { version = "==0.4.8", optional = true } beautifulsoup4 = { version = "04.12.3", optional = true } #gensim = { version = "==4.3.3", optional = true } scipy = { version = ">=1.7.0,<1.14.0", optional = true } -#scikit-learn = { version = "^1.4.2", optional = true } +scikit-learn = { version = "^1.4.2", optional = true } #spacy = { version = ">=3.0.0,<=3.8.2", optional = true } #transformers = { version = "^4.45.0", optional = true } #torch = { version = "^2.5.0", optional = true } @@ -68,6 +68,7 @@ nlp = [ ] nlp_tools = ["beautifulsoup4"] #ml_toolkits = ["gensim", "scipy", "scikit-learn"] +ml_toolkits = ["scikit-learn"] #spacy = ["spacy"] #transformers = ["transformers"] #torch = ["torch"] @@ -81,6 +82,7 @@ full = [ #"nltk", "textblob", "yake", "beautifulsoup4", + "scikit-learn", #"gensim", "scipy", "scikit-learn", #"spacy", #"transformers", From 45a416f930175b3cf05c16511b3e0890f805d26c Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:59:52 -0600 Subject: [PATCH 71/95] Create rag_issue_manager.py --- scripts/rag_issue_manager.py | 178 +++++++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 scripts/rag_issue_manager.py diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py new file mode 100644 index 000000000..0605fa690 --- /dev/null +++ b/scripts/rag_issue_manager.py @@ -0,0 +1,178 @@ +import os +import json +import requests +from swarmauri.utils.load_documents_from_folder import load_documents_from_folder +from swarmauri.llms.concrete.GroqModel import GroqModel +from swarmauri.agents.concrete.RagAgent import RagAgent +from swarmauri.conversations.concrete.Conversation import Conversation +from swarmauri.vector_stores.concrete.TfidfVectorStore import TfidfVectorStore +import argparse + +# GitHub API settings +GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") +REPO = os.getenv("REPO") +HEADERS = { + "Authorization": f"Bearer {GITHUB_TOKEN}", + "Accept": "application/vnd.github.v3+json", +} + +# GroqModel Initialization +GROQ_API_KEY = os.getenv("GROQ_API_KEY") +llm = GroqModel(api_key=GROQ_API_KEY) + +BASE_BRANCH = os.getenv("GITHUB_HEAD_REF") or os.getenv("GITHUB_REF", "unknown").split("/")[-1] +COMMIT_SHA = os.getenv("GITHUB_SHA", "unknown") +WORKFLOW_RUN_URL = os.getenv("GITHUB_SERVER_URL", "https://github.com") + f"/{REPO}/actions/runs/{os.getenv('GITHUB_RUN_ID', 'unknown')}" + +def parse_arguments(): + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description="Manage GitHub Issues for Test Failures") + parser.add_argument("--results-file", type=str, required=True, help="Path to the pytest JSON report file") + parser.add_argument("--package", type=str, required=True, help="Name of the matrix package where the error occurred.") + return parser.parse_args() + +def load_pytest_results(report_file): + """Load pytest results from the JSON report.""" + if not os.path.exists(report_file): + print(f"Report file not found: {report_file}") + return [] + + with open(report_file, "r") as f: + report = json.load(f) + + failures = [] + for test in report.get("tests", []): + if test.get("outcome") == "failed": + test_name = test.get("nodeid", "Unknown test") + failure_message = test.get("call", {}).get("longrepr", "No failure details available") + failures.append({ + "name": test_name, + "path": test.get("nodeid", "Unknown path"), + "message": failure_message + }) + return failures + +def ask_groq_for_fix(test_name, failure_message, stack_trace): + """Ask GroqModel for suggestions on fixing the test failure.""" + prompt = f""" + I have a failing test case named '{test_name}' in a Python project. The error message is: + {failure_message} + + The stack trace is: + {stack_trace} + + Can you help me identify the cause of this failure and suggest a fix? + """ + try: + current_directory = os.getcwd() + documents = load_documents_from_folder(folder_path=current_directory) + print(f"Loaded {len(documents)} documents.") + + # Step 3: Initialize the TFIDF Vector Store and add documents + vector_store = TfidfVectorStore() + vector_store.add_documents(documents) + print("Documents have been added to the TFIDF Vector Store.") + + # Step 5: Initialize the RagAgent with the vector store and language model + rag_agent = RagAgent(llm=llm, vector_store=vector_store, conversation=Conversation() top_k=20) + print("RagAgent initialized successfully.") + response = agent.exec(input_str=prompt) + return response + except Exception as e: + print(f"Error communicating with Groq: {e}") + return "Unable to retrieve suggestions from Groq at this time." + +def get_existing_issues(): + """Retrieve all existing issues with the pytest-failure label.""" + url = f"https://api.github.com/repos/{REPO}/issues" + params = {"labels": "pytest-failure", "state": "open"} + response = requests.get(url, headers=HEADERS, params=params) + response.raise_for_status() + return response.json() + +def create_issue(test, package): + """Create a new GitHub issue for the test failure.""" + groq_suggestion = ask_groq_for_fix(test["name"], test["message"], test["message"]) + url = f"https://api.github.com/repos/{REPO}/issues" + + # Construct the issue body + data = { + "title": f"[Test Case Failure]: {test['name']}", + "body": f""" +### Test Case: +{test['path']} + +### Failure Details: +{test['message']} + +--- + +### Suggested Fix (via Groq): +{groq_suggestion} + +--- + +### Context: +- **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) +- **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) +- **Commit Tree**: [{COMMIT_SHA}](https://github.com/{REPO}/tree/{COMMIT_SHA}) +- **Workflow Run**: [View Run]({WORKFLOW_RUN_URL}) +- **Matrix Package**: `{package}` + +### Labels: +This issue is auto-labeled for the `{package}` package. +""", + "labels": ["pytest-failure", package] + } + response = requests.post(url, headers=HEADERS, json=data) + response.raise_for_status() + print(f"Issue created for {test['name']} with Groq suggestion in package '{package}'.") + +def add_comment_to_issue(issue_number, test, package): + """Add a comment to an existing GitHub issue.""" + url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" + data = {"body": f""" +New failure detected: + +### Test Case: +{test['path']} + +### Details: +{test['message']} + +--- + +### Context: +- **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) +- **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) +- **Commit Tree**: [{COMMIT_SHA}](https://github.com/{REPO}/tree/{COMMIT_SHA}) +- **Workflow Run**: [View Run]({WORKFLOW_RUN_URL}) +- **Matrix Package**: `{package}` +"""} + response = requests.post(url, headers=HEADERS, json=data) + response.raise_for_status() + print(f"Comment added to issue {issue_number} for {test['name']}.") + +def process_failures(report_file, package): + """Process pytest failures and manage GitHub issues.""" + failures = load_pytest_results(report_file) + if not failures: + print("No test failures found.") + return + + existing_issues = get_existing_issues() + + for test in failures: + issue_exists = False + for issue in existing_issues: + if test["name"] in issue["title"]: + add_comment_to_issue(issue["number"], test, package) + issue_exists = True + break + + if not issue_exists: + create_issue(test, package) + +if __name__ == "__main__": + args = parse_arguments() + process_failures(args.results_file, args.package) From 7a1cabd1622849f44f9ee6179f4b78e187f1384e Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:05:45 -0600 Subject: [PATCH 72/95] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 0605fa690..3541e5803 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -54,7 +54,9 @@ def load_pytest_results(report_file): def ask_groq_for_fix(test_name, failure_message, stack_trace): """Ask GroqModel for suggestions on fixing the test failure.""" + system_context = f"Utilizing the following codebase solve the user's problem.\nCodebase:" prompt = f""" + \n\nUser Problem: I have a failing test case named '{test_name}' in a Python project. The error message is: {failure_message} @@ -74,9 +76,10 @@ def ask_groq_for_fix(test_name, failure_message, stack_trace): print("Documents have been added to the TFIDF Vector Store.") # Step 5: Initialize the RagAgent with the vector store and language model - rag_agent = RagAgent(llm=llm, vector_store=vector_store, conversation=Conversation() top_k=20) + rag_agent = RagAgent(system_context=system_context, + llm=llm, vector_store=vector_store, conversation=Conversation(), top_k=20) print("RagAgent initialized successfully.") - response = agent.exec(input_str=prompt) + response = rag_agent.exec(input_str=prompt, llm_kwargs={"max_tokens": 1750, "temperature":0.7}) return response except Exception as e: print(f"Error communicating with Groq: {e}") From 0fc76bc2e39d07756af9b24edf6517c78e74cbe1 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:06:30 -0600 Subject: [PATCH 73/95] cicd - Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index f5c7efb92..b1c1c76a7 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -152,6 +152,6 @@ jobs: REPO: ${{ github.repository }} run: | cd pkgs/swarmauri # Change to the directory containing pyproject.toml - poetry run python ../../scripts/manage_issues.py --results-file=pytest_results.json --package=${{ matrix.package_tests.package }} + poetry run python ../../scripts/rag_issue_manager.py --results-file=pytest_results.json --package=${{ matrix.package_tests.package }} From 171836a3d2b8bec1b9fb487f0f7fea1e50297429 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:13:22 -0600 Subject: [PATCH 74/95] cicd - Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 3541e5803..827a60bc7 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -16,14 +16,15 @@ "Accept": "application/vnd.github.v3+json", } -# GroqModel Initialization -GROQ_API_KEY = os.getenv("GROQ_API_KEY") -llm = GroqModel(api_key=GROQ_API_KEY) - +# GitHub Metadata BASE_BRANCH = os.getenv("GITHUB_HEAD_REF") or os.getenv("GITHUB_REF", "unknown").split("/")[-1] COMMIT_SHA = os.getenv("GITHUB_SHA", "unknown") WORKFLOW_RUN_URL = os.getenv("GITHUB_SERVER_URL", "https://github.com") + f"/{REPO}/actions/runs/{os.getenv('GITHUB_RUN_ID', 'unknown')}" +# GroqModel Initialization +GROQ_API_KEY = os.getenv("GROQ_API_KEY") +llm = GroqModel(api_key=GROQ_API_KEY) + def parse_arguments(): """Parse command-line arguments.""" parser = argparse.ArgumentParser(description="Manage GitHub Issues for Test Failures") From fa838085302aa8f1f6a745ed93750a86a79696e8 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:16:44 -0600 Subject: [PATCH 75/95] cicd - Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 827a60bc7..5c5d004a5 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -68,7 +68,7 @@ def ask_groq_for_fix(test_name, failure_message, stack_trace): """ try: current_directory = os.getcwd() - documents = load_documents_from_folder(folder_path=current_directory) + documents = load_documents_from_folder(folder_path=current_directory, include_extensions=['.py', '.md', '.yaml', '.toml', '.json']) print(f"Loaded {len(documents)} documents.") # Step 3: Initialize the TFIDF Vector Store and add documents @@ -134,6 +134,7 @@ def create_issue(test, package): def add_comment_to_issue(issue_number, test, package): """Add a comment to an existing GitHub issue.""" + groq_suggestion = ask_groq_for_fix(test["name"], test["message"], test["message"]) url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" data = {"body": f""" New failure detected: @@ -141,11 +142,16 @@ def add_comment_to_issue(issue_number, test, package): ### Test Case: {test['path']} -### Details: +### Failure Details: {test['message']} --- +### Suggested Fix (via Groq): +{groq_suggestion} + +--- + ### Context: - **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) - **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) From 5591fa28e08a8103770ee580637f0ac75bad263b Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:19:03 -0600 Subject: [PATCH 76/95] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 5c5d004a5..9062768bd 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -68,7 +68,8 @@ def ask_groq_for_fix(test_name, failure_message, stack_trace): """ try: current_directory = os.getcwd() - documents = load_documents_from_folder(folder_path=current_directory, include_extensions=['.py', '.md', '.yaml', '.toml', '.json']) + documents = load_documents_from_folder(folder_path=current_directory, + include_extensions=['py', 'md', 'yaml', 'toml', 'json']) print(f"Loaded {len(documents)} documents.") # Step 3: Initialize the TFIDF Vector Store and add documents From 2d1223ec8d9c84e897e5b3d2fe33f8dedce679ef Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:21:12 -0600 Subject: [PATCH 77/95] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 9062768bd..cbfc098ce 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -79,9 +79,9 @@ def ask_groq_for_fix(test_name, failure_message, stack_trace): # Step 5: Initialize the RagAgent with the vector store and language model rag_agent = RagAgent(system_context=system_context, - llm=llm, vector_store=vector_store, conversation=Conversation(), top_k=20) + llm=llm, vector_store=vector_store, conversation=Conversation()) print("RagAgent initialized successfully.") - response = rag_agent.exec(input_str=prompt, llm_kwargs={"max_tokens": 1750, "temperature":0.7}) + response = rag_agent.exec(input_str=prompt, top_k=20, llm_kwargs={"max_tokens": 1750, "temperature":0.7}) return response except Exception as e: print(f"Error communicating with Groq: {e}") From e8c65a590372cc3882d893af36a32ba2f9bcc9c7 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:28:02 -0600 Subject: [PATCH 78/95] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index cbfc098ce..9393ffbb4 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -2,7 +2,7 @@ import json import requests from swarmauri.utils.load_documents_from_folder import load_documents_from_folder -from swarmauri.llms.concrete.GroqModel import GroqModel +from swarmauri.llms.concrete.DeepInfraModel import DeepInfraModel from swarmauri.agents.concrete.RagAgent import RagAgent from swarmauri.conversations.concrete.Conversation import Conversation from swarmauri.vector_stores.concrete.TfidfVectorStore import TfidfVectorStore @@ -21,9 +21,9 @@ COMMIT_SHA = os.getenv("GITHUB_SHA", "unknown") WORKFLOW_RUN_URL = os.getenv("GITHUB_SERVER_URL", "https://github.com") + f"/{REPO}/actions/runs/{os.getenv('GITHUB_RUN_ID', 'unknown')}" -# GroqModel Initialization -GROQ_API_KEY = os.getenv("GROQ_API_KEY") -llm = GroqModel(api_key=GROQ_API_KEY) +# DeepInfraModel Initialization +DEEPINFRA_API_KEY = os.getenv("DEEPINFRA_API_KEY") +llm = DeepInfraModel(api_key=DEEPINFRA_API_KEY, name="meta-llama/Meta-Llama-3.1-405B-Instruct") def parse_arguments(): """Parse command-line arguments.""" From 16b5ec9a57b6e731a6429dcde9e958b164fa9ffa Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:28:26 -0600 Subject: [PATCH 79/95] Update test_changed_files.yaml --- .github/workflows/test_changed_files.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_changed_files.yaml b/.github/workflows/test_changed_files.yaml index b1c1c76a7..8cdc38823 100644 --- a/.github/workflows/test_changed_files.yaml +++ b/.github/workflows/test_changed_files.yaml @@ -147,7 +147,7 @@ jobs: - name: Process test results and manage issues if: always() env: - GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} + DEEPINFRA_API_KEY: ${{ secrets.DEEPINFRA_API_KEY }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} run: | From 0d186841bd976aaeba15deda08a7a0635d15b41c Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:29:42 -0600 Subject: [PATCH 80/95] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 9393ffbb4..cfdde97ca 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -81,7 +81,7 @@ def ask_groq_for_fix(test_name, failure_message, stack_trace): rag_agent = RagAgent(system_context=system_context, llm=llm, vector_store=vector_store, conversation=Conversation()) print("RagAgent initialized successfully.") - response = rag_agent.exec(input_str=prompt, top_k=20, llm_kwargs={"max_tokens": 1750, "temperature":0.7}) + response = rag_agent.exec(input_data=prompt, top_k=20, llm_kwargs={"max_tokens": 1750, "temperature":0.7}) return response except Exception as e: print(f"Error communicating with Groq: {e}") From 845fce535fdeef5dcf00eed55a79b4891786d35e Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:31:20 -0600 Subject: [PATCH 81/95] Update SimpleConversationAgent.py --- .../swarmauri/agents/concrete/SimpleConversationAgent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/swarmauri/swarmauri/agents/concrete/SimpleConversationAgent.py b/pkgs/swarmauri/swarmauri/agents/concrete/SimpleConversationAgent.py index 9653cf9c6..3dcd0d4bf 100644 --- a/pkgs/swarmauri/swarmauri/agents/concrete/SimpleConversationAgent.py +++ b/pkgs/swarmauri/swarmauri/agents/concrete/SimpleConversationAgent.py @@ -16,7 +16,7 @@ class SimpleConversationAgent(AgentConversationMixin, AgentBase): def exec( self, - input_str: Optional[Union[str, List[contentItem]]] = "", + input_data: Optional[Union[str, List[contentItem]]] = "", llm_kwargs: Optional[Dict] = {}, ) -> Any: From 7065cc1871f6edcd2748dbc2b50efd10af50c90e Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:32:37 -0600 Subject: [PATCH 82/95] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index cfdde97ca..8fc3908a7 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -4,7 +4,7 @@ from swarmauri.utils.load_documents_from_folder import load_documents_from_folder from swarmauri.llms.concrete.DeepInfraModel import DeepInfraModel from swarmauri.agents.concrete.RagAgent import RagAgent -from swarmauri.conversations.concrete.Conversation import Conversation +from swarmauri.conversations.concrete.SystemContextConversation import SystemContextConversation from swarmauri.vector_stores.concrete.TfidfVectorStore import TfidfVectorStore import argparse @@ -79,7 +79,7 @@ def ask_groq_for_fix(test_name, failure_message, stack_trace): # Step 5: Initialize the RagAgent with the vector store and language model rag_agent = RagAgent(system_context=system_context, - llm=llm, vector_store=vector_store, conversation=Conversation()) + llm=llm, vector_store=vector_store, conversation=SystemContextConversation()) print("RagAgent initialized successfully.") response = rag_agent.exec(input_data=prompt, top_k=20, llm_kwargs={"max_tokens": 1750, "temperature":0.7}) return response From 81440662dab0ad92a553c136a8d787dbad3a09a3 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:34:38 -0600 Subject: [PATCH 83/95] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 8fc3908a7..81d3a1c5b 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -4,7 +4,7 @@ from swarmauri.utils.load_documents_from_folder import load_documents_from_folder from swarmauri.llms.concrete.DeepInfraModel import DeepInfraModel from swarmauri.agents.concrete.RagAgent import RagAgent -from swarmauri.conversations.concrete.SystemContextConversation import SystemContextConversation +from swarmauri.conversations.concrete.MaxSystemContextConversation import MaxSystemContextConversation from swarmauri.vector_stores.concrete.TfidfVectorStore import TfidfVectorStore import argparse @@ -79,7 +79,7 @@ def ask_groq_for_fix(test_name, failure_message, stack_trace): # Step 5: Initialize the RagAgent with the vector store and language model rag_agent = RagAgent(system_context=system_context, - llm=llm, vector_store=vector_store, conversation=SystemContextConversation()) + llm=llm, vector_store=vector_store, conversation=MaxSystemContextConversation()) print("RagAgent initialized successfully.") response = rag_agent.exec(input_data=prompt, top_k=20, llm_kwargs={"max_tokens": 1750, "temperature":0.7}) return response From d59c1ec4bb239bb838c5bf1f28564b3e38ab6aa0 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:40:00 -0600 Subject: [PATCH 84/95] cicd - Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 81d3a1c5b..a527a65ec 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -81,7 +81,8 @@ def ask_groq_for_fix(test_name, failure_message, stack_trace): rag_agent = RagAgent(system_context=system_context, llm=llm, vector_store=vector_store, conversation=MaxSystemContextConversation()) print("RagAgent initialized successfully.") - response = rag_agent.exec(input_data=prompt, top_k=20, llm_kwargs={"max_tokens": 1750, "temperature":0.7}) + response = rag_agent.exec(input_data=prompt, top_k=20, llm_kwargs={"max_tokens": 750, "temperature":0.7}) + print(f"\nPrompt: \n{prompt}\n\n", '-'*10, f"Response: {response}\n\n", '='*10, '\n') return response except Exception as e: print(f"Error communicating with Groq: {e}") From f28cdd3d2204f7aa64b7b30dbff84be5aa5ce31e Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 16:42:11 -0600 Subject: [PATCH 85/95] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index a527a65ec..71e40c9e9 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -53,8 +53,8 @@ def load_pytest_results(report_file): }) return failures -def ask_groq_for_fix(test_name, failure_message, stack_trace): - """Ask GroqModel for suggestions on fixing the test failure.""" +def ask_agent_for_fix(test_name, failure_message, stack_trace): + """Ask the LLM Model for suggestions on fixing the test failure.""" system_context = f"Utilizing the following codebase solve the user's problem.\nCodebase:" prompt = f""" \n\nUser Problem: @@ -85,8 +85,8 @@ def ask_groq_for_fix(test_name, failure_message, stack_trace): print(f"\nPrompt: \n{prompt}\n\n", '-'*10, f"Response: {response}\n\n", '='*10, '\n') return response except Exception as e: - print(f"Error communicating with Groq: {e}") - return "Unable to retrieve suggestions from Groq at this time." + print(f"Error communicating with LLM: {e}") + return "Unable to retrieve suggestions from LLM at this time." def get_existing_issues(): """Retrieve all existing issues with the pytest-failure label.""" @@ -98,7 +98,7 @@ def get_existing_issues(): def create_issue(test, package): """Create a new GitHub issue for the test failure.""" - groq_suggestion = ask_groq_for_fix(test["name"], test["message"], test["message"]) + suggestion = ask_agent_for_fix(test["name"], test["message"], test["message"]) url = f"https://api.github.com/repos/{REPO}/issues" # Construct the issue body @@ -113,8 +113,8 @@ def create_issue(test, package): --- -### Suggested Fix (via Groq): -{groq_suggestion} +### Suggested Fix (via Agent): +{suggestion} --- @@ -132,11 +132,11 @@ def create_issue(test, package): } response = requests.post(url, headers=HEADERS, json=data) response.raise_for_status() - print(f"Issue created for {test['name']} with Groq suggestion in package '{package}'.") + print(f"Issue created for {test['name']} with LLM suggestion in package '{package}'.") def add_comment_to_issue(issue_number, test, package): """Add a comment to an existing GitHub issue.""" - groq_suggestion = ask_groq_for_fix(test["name"], test["message"], test["message"]) + suggestion = ask_agent_for_fix(test["name"], test["message"], test["message"]) url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" data = {"body": f""" New failure detected: @@ -149,8 +149,8 @@ def add_comment_to_issue(issue_number, test, package): --- -### Suggested Fix (via Groq): -{groq_suggestion} +### Suggested Fix (via Agent): +{suggestion} --- From b8cb8acbf0f45fc2326bfe53adfbca9333d4a21c Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:02:41 -0600 Subject: [PATCH 86/95] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 71e40c9e9..ab0a7d753 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -100,6 +100,11 @@ def create_issue(test, package): """Create a new GitHub issue for the test failure.""" suggestion = ask_agent_for_fix(test["name"], test["message"], test["message"]) url = f"https://api.github.com/repos/{REPO}/issues" + if package == 'core' or 'community' or 'experimental': + package_name = f"swarmauri_{package}" + resource_kind, type_kind = test['name'].split('/')[2:4] + comp_file_url = f"pkgs/{package}/{package_name}/{resource_kind}/concrete/{type_kind}.py" + test_file_url = f"pkgs/{package}/{test['name']}" # Construct the issue body data = { @@ -119,6 +124,8 @@ def create_issue(test, package): --- ### Context: +- **Component**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{comp_file_url) +- **Test File**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{test_file_url) - **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) - **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) - **Commit Tree**: [{COMMIT_SHA}](https://github.com/{REPO}/tree/{COMMIT_SHA}) @@ -138,6 +145,11 @@ def add_comment_to_issue(issue_number, test, package): """Add a comment to an existing GitHub issue.""" suggestion = ask_agent_for_fix(test["name"], test["message"], test["message"]) url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" + if package == 'core' or 'community' or 'experimental': + package_name = f"swarmauri_{package}" + resource_kind, type_kind = test['name'].split('/')[2:4] + comp_file_url = f"pkgs/{package}/{package_name}/{resource_kind}/concrete/{type_kind}.py" + test_file_url = f"pkgs/{package}/{test['name']}" data = {"body": f""" New failure detected: @@ -155,6 +167,8 @@ def add_comment_to_issue(issue_number, test, package): --- ### Context: +- **Component**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{comp_file_url) +- **Test File**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{test_file_url) - **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) - **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) - **Commit Tree**: [{COMMIT_SHA}](https://github.com/{REPO}/tree/{COMMIT_SHA}) From 37b148f270e12e671e5a550a049fe269f06ed626 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:03:55 -0600 Subject: [PATCH 87/95] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index ab0a7d753..e8e56715e 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -124,8 +124,8 @@ def create_issue(test, package): --- ### Context: -- **Component**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{comp_file_url) -- **Test File**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{test_file_url) +- **Component**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{comp_file_url}) +- **Test File**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{test_file_url}) - **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) - **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) - **Commit Tree**: [{COMMIT_SHA}](https://github.com/{REPO}/tree/{COMMIT_SHA}) @@ -167,8 +167,8 @@ def add_comment_to_issue(issue_number, test, package): --- ### Context: -- **Component**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{comp_file_url) -- **Test File**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{test_file_url) +- **Component**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{comp_file_url}) +- **Test File**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{test_file_url}) - **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) - **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) - **Commit Tree**: [{COMMIT_SHA}](https://github.com/{REPO}/tree/{COMMIT_SHA}) From 9e72c45dd369105b6ddcfc3295fbeb48889d546f Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:11:08 -0600 Subject: [PATCH 88/95] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index e8e56715e..e3a5408c9 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -103,8 +103,8 @@ def create_issue(test, package): if package == 'core' or 'community' or 'experimental': package_name = f"swarmauri_{package}" resource_kind, type_kind = test['name'].split('/')[2:4] - comp_file_url = f"pkgs/{package}/{package_name}/{resource_kind}/concrete/{type_kind}.py" - test_file_url = f"pkgs/{package}/{test['name']}" + comp_file_url = f"pkgs/{package}/{package_name}/{resource_kind}/concrete/{type_kind.split('_')[0]}.py" + test_file_url = f"pkgs/{package}/{test['name'].split('::')[0]}" # Construct the issue body data = { @@ -147,9 +147,9 @@ def add_comment_to_issue(issue_number, test, package): url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" if package == 'core' or 'community' or 'experimental': package_name = f"swarmauri_{package}" - resource_kind, type_kind = test['name'].split('/')[2:4] - comp_file_url = f"pkgs/{package}/{package_name}/{resource_kind}/concrete/{type_kind}.py" - test_file_url = f"pkgs/{package}/{test['name']}" + resource_kind, type_kinds = test['name'].split('/')[2:4] + comp_file_url = f"pkgs/{package}/{package_name}/{resource_kind}/concrete/{type_kind.split('_')[0]}.py" + test_file_url = f"pkgs/{package}/{test['name'].split('::')[0]}" data = {"body": f""" New failure detected: From fb04556044e9a5ce349c7f41052728919b0d36e8 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:13:56 -0600 Subject: [PATCH 89/95] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index e3a5408c9..3a8d7f4f9 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -147,7 +147,7 @@ def add_comment_to_issue(issue_number, test, package): url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" if package == 'core' or 'community' or 'experimental': package_name = f"swarmauri_{package}" - resource_kind, type_kinds = test['name'].split('/')[2:4] + resource_kind, type_kind = test['name'].split('/')[2:4] comp_file_url = f"pkgs/{package}/{package_name}/{resource_kind}/concrete/{type_kind.split('_')[0]}.py" test_file_url = f"pkgs/{package}/{test['name'].split('::')[0]}" data = {"body": f""" From 951706ce965af6e318fbd7159bfc413725de0b81 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:15:45 -0600 Subject: [PATCH 90/95] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 3a8d7f4f9..41708a7a6 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -55,10 +55,10 @@ def load_pytest_results(report_file): def ask_agent_for_fix(test_name, failure_message, stack_trace): """Ask the LLM Model for suggestions on fixing the test failure.""" - system_context = f"Utilizing the following codebase solve the user's problem.\nCodebase:" + system_context = f"Utilizing the following codebase solve the user's problem.\nCodebase:\n\n" prompt = f""" \n\nUser Problem: - I have a failing test case named '{test_name}' in a Python project. The error message is: + I have a failing pytest test case named '{test_name}' in a Python project. The error message is: {failure_message} The stack trace is: From 3a7467f78c92822b9da6ead4f2852eb13eb27540 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:18:02 -0600 Subject: [PATCH 91/95] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 41708a7a6..ccb4721d7 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -102,6 +102,8 @@ def create_issue(test, package): url = f"https://api.github.com/repos/{REPO}/issues" if package == 'core' or 'community' or 'experimental': package_name = f"swarmauri_{package}" + else: + package_name = "swarmauri" resource_kind, type_kind = test['name'].split('/')[2:4] comp_file_url = f"pkgs/{package}/{package_name}/{resource_kind}/concrete/{type_kind.split('_')[0]}.py" test_file_url = f"pkgs/{package}/{test['name'].split('::')[0]}" @@ -146,7 +148,9 @@ def add_comment_to_issue(issue_number, test, package): suggestion = ask_agent_for_fix(test["name"], test["message"], test["message"]) url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" if package == 'core' or 'community' or 'experimental': - package_name = f"swarmauri_{package}" + package_name = f"swarmauri_{package}" + else: + package_name = "swarmauri" resource_kind, type_kind = test['name'].split('/')[2:4] comp_file_url = f"pkgs/{package}/{package_name}/{resource_kind}/concrete/{type_kind.split('_')[0]}.py" test_file_url = f"pkgs/{package}/{test['name'].split('::')[0]}" From 832e3669b9b3fbeacf9fb628d6615225ece983c6 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:21:54 -0600 Subject: [PATCH 92/95] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index ccb4721d7..2bbde6d5c 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -100,7 +100,7 @@ def create_issue(test, package): """Create a new GitHub issue for the test failure.""" suggestion = ask_agent_for_fix(test["name"], test["message"], test["message"]) url = f"https://api.github.com/repos/{REPO}/issues" - if package == 'core' or 'community' or 'experimental': + if package in {'core', 'community', 'experimental'}: package_name = f"swarmauri_{package}" else: package_name = "swarmauri" @@ -147,7 +147,7 @@ def add_comment_to_issue(issue_number, test, package): """Add a comment to an existing GitHub issue.""" suggestion = ask_agent_for_fix(test["name"], test["message"], test["message"]) url = f"https://api.github.com/repos/{REPO}/issues/{issue_number}/comments" - if package == 'core' or 'community' or 'experimental': + if package in {'core', 'community', 'experimental'}: package_name = f"swarmauri_{package}" else: package_name = "swarmauri" From 24ae1f0b45ef1c676a7cd695fe7bba8b53beee32 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:28:21 -0600 Subject: [PATCH 93/95] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 2bbde6d5c..9e803698e 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -126,13 +126,13 @@ def create_issue(test, package): --- ### Context: -- **Component**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{comp_file_url}) -- **Test File**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{test_file_url}) +- **Matrix Package**: [{package}](https://github.com/{REPO}/tree/{BASE_BRANCH}/pkgs/{package}) +- **Component**: [{comp_file_url}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{comp_file_url}) +- **Test File**: [{test_file_url}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{test_file_url}) - **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) - **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) - **Commit Tree**: [{COMMIT_SHA}](https://github.com/{REPO}/tree/{COMMIT_SHA}) - **Workflow Run**: [View Run]({WORKFLOW_RUN_URL}) -- **Matrix Package**: `{package}` ### Labels: This issue is auto-labeled for the `{package}` package. @@ -171,13 +171,14 @@ def add_comment_to_issue(issue_number, test, package): --- ### Context: -- **Component**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{comp_file_url}) -- **Test File**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{test_file_url}) +- **Matrix Package**: [{package}](https://github.com/{REPO}/tree/{BASE_BRANCH}/pkgs/{package}) +- **Component**: [{comp_file_url}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{comp_file_url}) +- **Test File**: [{test_file_url}](https://github.com/{REPO}/tree/{BASE_BRANCH}/{test_file_url}) - **Branch**: [{BASE_BRANCH}](https://github.com/{REPO}/tree/{BASE_BRANCH}) - **Commit**: [{COMMIT_SHA}](https://github.com/{REPO}/commit/{COMMIT_SHA}) - **Commit Tree**: [{COMMIT_SHA}](https://github.com/{REPO}/tree/{COMMIT_SHA}) - **Workflow Run**: [View Run]({WORKFLOW_RUN_URL}) -- **Matrix Package**: `{package}` + """} response = requests.post(url, headers=HEADERS, json=data) response.raise_for_status() From 54b6e77c27409ff0183179e180f0ad0d49161a0b Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:30:16 -0600 Subject: [PATCH 94/95] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 9e803698e..683c020eb 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -113,7 +113,7 @@ def create_issue(test, package): "title": f"[Test Case Failure]: {test['name']}", "body": f""" ### Test Case: -{test['path']} +`{test['path']}` ### Failure Details: {test['message']} @@ -158,7 +158,7 @@ def add_comment_to_issue(issue_number, test, package): New failure detected: ### Test Case: -{test['path']} +`{test['path']}` ### Failure Details: {test['message']} From 32b599d7afdef86b83c2a0a25e371ce254e08894 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:36:10 -0600 Subject: [PATCH 95/95] Update rag_issue_manager.py --- scripts/rag_issue_manager.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/rag_issue_manager.py b/scripts/rag_issue_manager.py index 683c020eb..3581fecbf 100644 --- a/scripts/rag_issue_manager.py +++ b/scripts/rag_issue_manager.py @@ -116,7 +116,9 @@ def create_issue(test, package): `{test['path']}` ### Failure Details: +```python {test['message']} +``` --- @@ -161,7 +163,9 @@ def add_comment_to_issue(issue_number, test, package): `{test['path']}` ### Failure Details: +```python {test['message']} +``` ---