Merge branch 'main' into offlineAgentEval

microsoft · Apr 20, 2024 · 5a3969a · 5a3969a
2 parents 2d6658a + ded2d61
commit 5a3969a
Show file tree

Hide file tree

Showing 508 changed files with 48,962 additions and 11,858 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -9,18 +9,24 @@ FROM mcr.microsoft.com/vscode/devcontainers/python:3.10
 # Update the OS and maybe install packages
 #
 ENV DEBIAN_FRONTEND=noninteractive
+
+# add git lhs to apt
+RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
+
 RUN apt-get update \
    && apt-get upgrade -y  \
-   && apt-get -y install --no-install-recommends build-essential npm \
+   && apt-get -y install --no-install-recommends build-essential npm git-lfs \
    && apt-get autoremove -y \
    && apt-get clean -y \
-   && wget https://github.com/quarto-dev/quarto-cli/releases/download/v1.5.23/quarto-1.5.23-linux-amd64.deb \
-   && dpkg -i quarto-1.5.23-linux-amd64.deb \
-   && rm -rf /var/lib/apt/lists/* quarto-1.5.23-linux-amd64.deb
+   && arch=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) \
+   && wget https://github.com/quarto-dev/quarto-cli/releases/download/v1.5.23/quarto-1.5.23-linux-${arch}.deb \
+   && dpkg -i quarto-1.5.23-linux-${arch}.deb \
+   && rm -rf /var/lib/apt/lists/* quarto-1.5.23-linux-${arch}.deb
 ENV DEBIAN_FRONTEND=dialog
 
 # For docs
 RUN npm install --global yarn
+RUN pip install --upgrade pip
 RUN pip install pydoc-markdown
 RUN pip install pyyaml
 RUN pip install colored
diff --git a/.devcontainer/dev/Dockerfile b/.devcontainer/dev/Dockerfile
@@ -1,10 +1,13 @@
 # Basic setup
 FROM python:3.11-slim-bookworm
 
+# add git lhs to apt
+RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
+
 # Update and install necessary packages
 RUN apt-get update && apt-get -y update
 # added vim and nano for convenience
-RUN apt-get install -y sudo git npm vim nano curl wget
+RUN apt-get install -y sudo git npm vim nano curl wget git-lfs
 
 # Setup a non-root user 'autogen' with sudo access
 RUN adduser --disabled-password --gecos '' autogen
@@ -44,6 +47,7 @@ ENV PATH="${PATH}:/home/autogen/quarto/quarto-1.5.23/bin/"
 EXPOSE 3000
 
 # Pre-load popular Python packages
+RUN pip install --upgrade pip
 RUN pip install numpy pandas matplotlib seaborn scikit-learn requests urllib3 nltk pillow pytest beautifulsoup4
 
 # Set the default command to bash

diff --git a/.devcontainer/full/Dockerfile b/.devcontainer/full/Dockerfile
@@ -1,9 +1,12 @@
 FROM python:3.11-slim-bookworm
 
+# add git lhs to apt
+RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
+
 # Update and install dependencies
 RUN apt-get update \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-        software-properties-common sudo\
+        software-properties-common sudo git-lfs \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 

diff --git a/.devcontainer/studio/Dockerfile b/.devcontainer/studio/Dockerfile
@@ -9,14 +9,19 @@ FROM mcr.microsoft.com/vscode/devcontainers/python:3.10
 # Update the OS and maybe install packages
 #
 ENV DEBIAN_FRONTEND=noninteractive
+
+# add git lhs to apt
+RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
+
 RUN apt-get update \
    && apt-get upgrade -y  \
-   && apt-get -y install --no-install-recommends build-essential npm \
+   && apt-get -y install --no-install-recommends build-essential npm git-lfs \
    && apt-get autoremove -y \
    && apt-get clean -y \
    && rm -rf /var/lib/apt/lists/*
 ENV DEBIAN_FRONTEND=dialog
 
 # For docs
 RUN npm install --global yarn
+RUN pip install --upgrade pip
 RUN pip install pydoc-markdown
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1,3 @@
+*.png filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -31,9 +31,9 @@ jobs:
         os: [ubuntu-latest, macos-latest, windows-latest]
         python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install packages and dependencies
@@ -43,10 +43,10 @@ jobs:
           python -c "import autogen"
           pip install pytest mock
       - name: Install optional dependencies for code executors
-        # code executors auto skip without deps, so only run for python 3.11
+        # code executors and udfs auto skip without deps, so only run for python 3.11
         if: matrix.python-version == '3.11'
         run: |
-          pip install -e ".[jupyter-executor]"
+          pip install -e ".[jupyter-executor,test]"
           python -m ipykernel install --user --name python3
       - name: Set AUTOGEN_USE_DOCKER based on OS
         shell: bash
@@ -57,16 +57,16 @@ jobs:
       - name: Test with pytest skipping openai tests
         if: matrix.python-version != '3.10' && matrix.os == 'ubuntu-latest'
         run: |
-          pytest test --skip-openai
+          pytest test --ignore=test/agentchat/contrib --skip-openai --durations=10 --durations-min=1.0
       - name: Test with pytest skipping openai and docker tests
         if: matrix.python-version != '3.10' && matrix.os != 'ubuntu-latest'
         run: |
-          pytest test --skip-openai --skip-docker
+          pytest test --ignore=test/agentchat/contrib --skip-openai --skip-docker --durations=10 --durations-min=1.0
       - name: Coverage
         if: matrix.python-version == '3.10'
         run: |
-          pip install -e .[test,redis]
-          coverage run -a -m pytest test --ignore=test/agentchat/contrib --skip-openai
+          pip install -e .[test,redis,websockets]
+          coverage run -a -m pytest test --ignore=test/agentchat/contrib --skip-openai --durations=10 --durations-min=1.0
           coverage xml
       - name: Upload coverage to Codecov
         if: matrix.python-version == '3.10'

diff --git a/.github/workflows/contrib-openai.yml b/.github/workflows/contrib-openai.yml
@@ -27,11 +27,11 @@ jobs:
     steps:
       # checkout to pr branch
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install packages and dependencies
@@ -41,19 +41,23 @@ jobs:
           pip install -e .
           python -c "import autogen"
           pip install coverage pytest-asyncio
+      - name: Install PostgreSQL
+        run: |
+          sudo apt install postgresql -y
+      - name: Start PostgreSQL service
+        run: sudo service postgresql start
       - name: Install packages for test when needed
         run: |
           pip install docker
-          pip install qdrant_client[fastembed]
-          pip install -e .[retrievechat]
+          pip install -e .[retrievechat-qdrant,retrievechat-pgvector]
       - name: Coverage
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
           AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
           OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }}
         run: |
-          coverage run -a -m pytest test/agentchat/contrib/test_retrievechat.py test/agentchat/contrib/test_qdrant_retrievechat.py
+          coverage run -a -m pytest test/agentchat/contrib/test_retrievechat.py::test_retrievechat test/agentchat/contrib/test_qdrant_retrievechat.py::test_retrievechat test/agentchat/contrib/test_pgvector_retrievechat.py::test_retrievechat
           coverage xml
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
@@ -70,11 +74,11 @@ jobs:
     steps:
       # checkout to pr branch
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install packages and dependencies
@@ -111,11 +115,11 @@ jobs:
     steps:
       # checkout to pr branch
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install packages and dependencies
@@ -152,11 +156,11 @@ jobs:
     steps:
       # checkout to pr branch
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install packages and dependencies
@@ -190,11 +194,11 @@ jobs:
     steps:
       # checkout to pr branch
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install packages and dependencies
@@ -231,11 +235,11 @@ jobs:
     steps:
       # checkout to pr branch
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install packages and dependencies
@@ -270,11 +274,11 @@ jobs:
       steps:
         # checkout to pr branch
         - name: Checkout
-          uses: actions/checkout@v3
+          uses: actions/checkout@v4
           with:
             ref: ${{ github.event.pull_request.head.sha }}
         - name: Set up Python ${{ matrix.python-version }}
-          uses: actions/setup-python@v4
+          uses: actions/setup-python@v5
           with:
             python-version: ${{ matrix.python-version }}
         - name: Install packages and dependencies
@@ -299,3 +303,77 @@ jobs:
           with:
             file: ./coverage.xml
             flags: unittests
+  ImageGen:
+      strategy:
+        matrix:
+          os: [ubuntu-latest]
+          python-version: ["3.12"]
+      runs-on: ${{ matrix.os }}
+      environment: openai1
+      steps:
+        # checkout to pr branch
+        - name: Checkout
+          uses: actions/checkout@v4
+          with:
+            ref: ${{ github.event.pull_request.head.sha }}
+        - name: Set up Python ${{ matrix.python-version }}
+          uses: actions/setup-python@v5
+          with:
+            python-version: ${{ matrix.python-version }}
+        - name: Install packages and dependencies
+          run: |
+            docker --version
+            python -m pip install --upgrade pip wheel
+            pip install -e .[lmm]
+            python -c "import autogen"
+            pip install coverage pytest
+        - name: Coverage
+          env:
+            OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          run: |
+            coverage run -a -m pytest test/agentchat/contrib/capabilities/test_image_generation_capability.py
+            coverage xml
+        - name: Upload coverage to Codecov
+          uses: codecov/codecov-action@v3
+          with:
+            file: ./coverage.xml
+            flags: unittests
+
+  AgentOptimizer:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+        python-version: [ "3.11" ]
+    runs-on: ${{ matrix.os }}
+    environment: openai1
+    steps:
+      # checkout to pr branch
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install packages and dependencies
+        run: |
+          docker --version
+          python -m pip install --upgrade pip wheel
+          pip install -e .
+          python -c "import autogen"
+          pip install coverage pytest
+      - name: Coverage
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
+          AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
+          OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }}
+        run: |
+          coverage run -a -m pytest test/agentchat/contrib/test_agent_optimizer.py
+          coverage xml
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./coverage.xml
+          flags: unittests