Merge a8aa238 into 764fed6

.github/workflows/increment_version_dev.yaml

@@ -24,11 +24,11 @@ jobs:
       - name: Increment versions in pyproject.toml
         run: |
           set -x
-          
+
           echo "Incrementing versions..."
           find . -name "pyproject.toml" | while read -r pyproject; do
               echo "Processing $pyproject"
-          
+
               # Extract current version
               CURRENT_VERSION=$(python -c "
           import tomlkit
@@ -43,31 +43,30 @@ jobs:
               print(f'Error reading version from {pyproject}: {e}', end='')
               exit(1)
           ")
-          
+
               echo "Extracted CURRENT_VERSION: $CURRENT_VERSION"
-          
+
               if [ -z "$CURRENT_VERSION" ]; then
                   echo "Error: Could not extract the current version from $pyproject"
                   cat "$pyproject"
                   continue
               fi
-          
+
               # Increment version
               BASE_VERSION=$(echo "$CURRENT_VERSION" | sed -E 's/(.*)-dev.*/\1/')
               DEV_PART=$(echo "$CURRENT_VERSION" | grep -oE 'dev[0-9]+$' | grep -oE '[0-9]+')
-          
+
+              # Fallback if no DEV_PART is found
               if [ -z "$DEV_PART" ]; then
                   DEV_PART=0
               fi
-          
+
               NEW_DEV_PART=$((DEV_PART + 1))
               NEW_VERSION="${BASE_VERSION}-dev${NEW_DEV_PART}"
-          
+
               echo "Updating version from $CURRENT_VERSION to $NEW_VERSION"
           done
 
-
-
       - name: Commit changes
         run: |
           git config user.name "github-actions[bot]"

.github/workflows/sequence_publish.yaml

@@ -93,7 +93,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Wait for swarmauri
-        run: sleep 60
+        run: sleep 120
 
       - name: Set up Python 3.12
         uses: actions/setup-python@v5

.github/workflows/test_changed_files.yaml

@@ -99,6 +99,9 @@ jobs:
   run-tests:
     needs: detect-changed-files
     runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      contents: read
     if: ${{ needs.detect-changed-files.outputs.matrix != '[]' }}
     strategy:
       fail-fast: false
@@ -134,9 +137,32 @@ jobs:
           cd pkgs/${{ matrix.package_tests.package }}
           poetry install --no-cache --all-extras -vv
 
-      - name: Run all tests for the package
+      - name: Run tests and save results
         run: |
           echo "Running tests for package: ${{ matrix.package_tests.package }}"
           echo "Test files: ${{ matrix.package_tests.tests }}"
           cd pkgs/${{ matrix.package_tests.package }}
-          poetry run pytest ${{ matrix.package_tests.tests }}
+          poetry run pytest -v ${{ matrix.package_tests.tests }} -n 4 --dist=loadfile --tb=short --json-report --json-report-file=pytest_results.json || true
+
+    
+      - name: Output test results for debugging
+        run: |
+          ls -R
+          cat pkgs/${{ matrix.package_tests.package }}/pytest_results.json
+
+      # - name: Classify test results
+      #   run: |
+      #     python scripts/classify_results.py pkgs/${{ matrix.package_tests.package }}/results.xml
+      #   continue-on-error: false
+
+      - name: Process test results and manage issues
+        if: |
+          github.event.pull_request.head.repo.full_name == github.repository && always()
+        env:
+          DEEPINFRA_API_KEY: ${{ secrets.DEEPINFRA_API_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+        run: |
+          cd pkgs/swarmauri  # Change to the directory containing pyproject.toml
+          poetry run python ../../scripts/rag_issue_manager.py --results-file=pytest_results.json --package=${{ matrix.package_tests.package }}
+          

pkgs/community/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "swarmauri-community"
-version = "0.5.2.dev20"
+version = "0.5.3.dev5"
 description = "This repository includes Swarmauri community components."
 authors = ["Jacob Stewart <jacob@swarmauri.com>"]
 license = "Apache-2.0"
@@ -15,48 +15,74 @@ classifiers = [
 
 [tool.poetry.dependencies]
 python = ">=3.10,<3.13"
-captcha = "*"
-chromadb = "*"
-duckdb = "*"
-folium = "*"
-gensim = "*"
-#google-generativeai = "*"
-gradio = "*"
-leptonai = "0.22.0"
-neo4j = "*"
-nltk = "*"
-#openai = "^1.52.0"
-pandas = "*"
-psutil = "*"
-pygithub = "*"
-python-dotenv = "*"
-qrcode = "*"
-redis = "^4.0"
-scikit-learn="^1.4.2"
-swarmauri = "==0.5.2"
-textstat = "*"
-transformers = ">=4.45.0"
-typing_extensions = "*"
-tiktoken = "*"
-pymupdf = "*"
-annoy = "*"
-qdrant_client = "*"
-weaviate = "*"
-pinecone-client = { version = "*", extras = ["grpc"] }
-PyPDF2 = "*"
-pypdftk = "*"
-weaviate-client = "*"
-protobuf = "^3.20.0"
-# Pacmap requires specific version of numba
-#numba = ">=0.59.0"
-#pacmap = "==0.7.3"
+captcha = "^0.6.0"
+
+# We should remove and only rely on httpx
+requests = "^2.32.3"
+
+chromadb = { version = "^0.5.17", optional = true }
+duckdb = { version = "^1.1.1", optional = true }
+folium = { version = "^0.18.0", optional = true }
+gensim = { version = "^4.3.3", optional = true }
+gradio = { version = "^5.4.0", optional = true }
+leptonai = { version = "^0.22.0", optional = true }
+neo4j = { version = "^5.25.0", optional = true }
+nltk = { version = "^3.9.1", optional = true }
+pandas = "^2.2.3"
+psutil = { version = "^6.1.0", optional = true }
+pygithub = { version = "^2.4.0", optional = true }
+qrcode = { version = "^8.0", optional = true }
+redis = { version = "^4.0", optional = true }
+swarmauri = "==0.5.3.dev5"
+textstat = { version = "^0.7.4", optional = true }
+transformers = { version = ">=4.45.0", optional = true }
+typing_extensions = "^4.12.2"
+tiktoken = { version = "^0.8.0", optional = true }
+PyMuPDF = { version = "^1.24.12", optional = true }
+annoy = { version = "^1.17.3", optional = true }
+qdrant-client = { version = "^1.12.0", optional = true }
+pinecone-client = { version = "^5.0.1", optional = true, extras = ["grpc"] }
+pypdf = { version = "^5.0.1", optional = true }
+pypdftk = { version = "^0.5", optional = true }
+weaviate-client = { version = "^4.9.2", optional = true }
+textblob = { version = "^0.18.0", optional = true }
+torch = { version = "^2.4.1", optional = true}
+scikit-learn = { version = "^1.5.2", optional = true }
+#protobuf = { version = "^3.20.0", optional = true }
+
+[tool.poetry.extras]
+# Grouped optional dependencies
+nlp = ["nltk", "gensim", "textstat", "textblob", "torch", "scikit-learn"]
+ml_toolkits = ["transformers", "annoy"]
+visualization = ["folium"]
+storage = ["redis", "duckdb", "neo4j", "chromadb", "qdrant-client", "weaviate-client", "pinecone-client"]
+document_processing = ["pypdf", "PyMuPDF", "pypdftk"]
+cloud_integration = ["psutil", "qrcode", "pygithub"]
+gradio = ["gradio"]
+model_clients = ["leptonai"]
+tiktoken = ["tiktoken"]
 
+# Full installation
+full = [
+    "nltk", "gensim", "textstat", "textblob", "torch", "scikit-learn",
+    "transformers", "annoy",
+    "folium",
+    "redis", "duckdb", "neo4j", "chromadb", "qdrant-client", "weaviate-client", "pinecone-client",
+    "pypdf", "PyMuPDF", "pypdftk",
+    "psutil", "qrcode", "pygithub",
+    "gradio",
+    "leptonai",
+    "tiktoken"
+]
 
 [tool.poetry.dev-dependencies]
-flake8 = "^7.0"  # Add flake8 as a development dependency
-pytest = "^8.0"  # Ensure pytest is also added if you run tests
+flake8 = "^7.0"
+pytest = "^8.0"
 pytest-asyncio = ">=0.24.0"
 pytest-xdist = "^3.6.1"
+pytest-json-report = "^1.5.0"
+python-dotenv = "*"
+requests = "^2.32.3"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]
@@ -70,12 +96,10 @@ markers = [
     "unit: Unit tests",
     "integration: Integration tests",
     "acceptance: Acceptance tests",
-    "experimental: Experimental tests",
+    "experimental: Experimental tests"
 ]
-
 log_cli = true
 log_cli_level = "INFO"
 log_cli_format = "%(asctime)s [%(levelname)s] %(message)s"
 log_cli_date_format = "%Y-%m-%d %H:%M:%S"
-
 asyncio_default_fixture_loop_scope = "function"

pkgs/community/swarmauri_community/document_stores/concrete/__init__.py

@@ -1,3 +1,10 @@
-from swarmauri_community.document_stores.concrete.RedisDocumentStore import (
-    RedisDocumentStore,
-)
+from swarmauri.utils._lazy_import import _lazy_import
+
+documents_stores_files = [
+    ("swarmauri_community.documents_stores.concrete.RedisDocumentStore", "RedisDocumentStore"),
+]
+
+for module_name, class_name in documents_stores_files:
+    globals()[class_name] = _lazy_import(module_name, class_name)
+
+__all__ = [class_name for _, class_name in documents_stores_files]

pkgs/community/swarmauri_community/embeddings/concrete/__init__.py

@@ -0,0 +1,12 @@
+from swarmauri.utils._lazy_import import _lazy_import
+
+
+embeddings_files = [
+    ("swarmauri_community.embeddings.concrete.Doc2VecEmbedding", "Doc2VecEmbedding"),
+    ("swarmauri_community.embeddings.concrete.MlmEmbedding", "MlmEmbedding"),
+]
+
+for module_name, class_name in embeddings_files:
+    globals()[class_name] = _lazy_import(module_name, class_name)
+
+__all__ = [class_name for _, class_name in embeddings_files]

pkgs/community/swarmauri_community/llms/concrete/__init__.py

@@ -1,4 +1,12 @@
-from swarmauri_community.llms.concrete.LeptonAIImgGenModel import LeptonAIImgGenModel
-from swarmauri_community.llms.concrete.LeptonAIModel import LeptonAIModel
+from swarmauri.utils._lazy_import import _lazy_import
 
-__all__ = ["LeptonAIImgGenModel", "LeptonAIModel"]
+llms_files = [
+    ("swarmauri_community.llms.concrete.LeptonAIImgGenModel", "LeptonAIImgGenModel"),
+    ("swarmauri_community.llms.concrete.LeptonAIModel", "LeptonAIModel"),
+    ("swarmauri_community.llms.concrete.PytesseractImg2TextModel", "PytesseractImg2TextModel"),
+]
+
+for module_name, class_name in llms_files:
+    globals()[class_name] = _lazy_import(module_name, class_name)
+
+__all__ = [class_name for _, class_name in llms_files]

pkgs/community/swarmauri_community/measurements/concrete/__init__.py

@@ -1,6 +1,11 @@
-from swarmauri_community.measurements.concrete.MutualInformationMeasurement import (
-    MutualInformationMeasurement,
-)
-from swarmauri_community.measurements.concrete.TokenCountEstimatorMeasurement import (
-    TokenCountEstimatorMeasurement,
-)
+from swarmauri.utils._lazy_import import _lazy_import
+
+measurement_files = [
+    ("swarmauri_community.measurements.concrete.MutualInformationMeasurement", "MutualInformationMeasurement"),
+    ("swarmauri_community.measurements.concrete.TokenCountEstimatorMeasurement", "TokenCountEstimatorMeasurement"),
+]
+
+for module_name, class_name in measurement_files:
+    globals()[class_name] = _lazy_import(module_name, class_name)
+
+__all__ = [class_name for _, class_name in measurement_files]

pkgs/community/swarmauri_community/parsers/concrete/__init__.py

@@ -1,3 +1,16 @@
-from swarmauri_community.parsers.concrete.FitzPdfParser import PDFtoTextParser
-from swarmauri_community.parsers.concrete.PyPDF2Parser import PyPDF2Parser
-from swarmauri_community.parsers.concrete.PyPDFTKParser import PyPDFTKParser
+from swarmauri.utils._lazy_import import _lazy_import
+
+parsers_files = [
+    ("swarmauri_community.parsers.concrete.BERTEmbeddingParser", "BERTEmbeddingParser"),
+    ("swarmauri_community.parsers.concrete.EntityRecognitionParser", "EntityRecognitionParser"),
+    ("swarmauri_community.parsers.concrete.FitzPdfParser", "FitzPdfParser"),
+    ("swarmauri_community.parsers.concrete.PyPDF2Parser", "PyPDF2Parser"),
+    ("swarmauri_community.parsers.concrete.PyPDFTKParser", "PyPDFTKParser"),
+    ("swarmauri_community.parsers.concrete.TextBlobNounParser", "TextBlobNounParser"),
+    ("swarmauri_community.parsers.concrete.TextBlobSentenceParser", "TextBlobSentenceParser"),
+]
+
+for module_name, class_name in parsers_files:
+    globals()[class_name] = _lazy_import(module_name, class_name)
+
+__all__ = [class_name for _, class_name in parsers_files]

pkgs/community/swarmauri_community/retrievers/concrete/__init__.py

@@ -1,5 +1,10 @@
-# -*- coding: utf-8 -*-
+from swarmauri.utils._lazy_import import _lazy_import
 
-from swarmauri_community.retrievers.concrete.RedisDocumentRetriever import (
-    RedisDocumentRetriever,
-)
+retriever_files = [
+    ("swarmauri_community.retrievers.concrete.RedisDocumentRetriever", "RedisDocumentRetriever"),
+]
+
+for module_name, class_name in retriever_files:
+    globals()[class_name] = _lazy_import(module_name, class_name)
+
+__all__ = [class_name for _, class_name in retriever_files]

pkgs/community/swarmauri_community/state/__init__.py

@@ -0,0 +1 @@
+from swarmauri_community.state.concrete import *