Merge pull request #852 from geekan/dev

version 0.7: merge dev

.github/workflows/fulltest.yaml

@@ -0,0 +1,84 @@
+name: Full Tests
+
+on:
+  workflow_dispatch:
+  pull_request_target:
+  push: 
+    branches:
+      - 'main'
+      - 'dev'
+      - '*-release'
+      - '*-debugger'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    environment: unittest
+    strategy:
+      matrix:
+        # python-version: ['3.9', '3.10', '3.11']
+        python-version: ['3.9']
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+    - name: Install dependencies
+      run: |
+        sh tests/scripts/run_install_deps.sh
+    - name: Run reverse proxy script for ssh service
+      if: contains(github.ref, '-debugger')
+      continue-on-error: true
+      env:
+        FPR_SERVER_ADDR: ${{ secrets.FPR_SERVER_ADDR }}
+        FPR_TOKEN: ${{ secrets.FPR_TOKEN }}
+        FPR_SSH_REMOTE_PORT: ${{ secrets.FPR_SSH_REMOTE_PORT }}
+        RSA_PUB: ${{ secrets.RSA_PUB }}
+        SSH_PORT: ${{ vars.SSH_PORT || '22'}}
+      run: |
+        echo "Run \"ssh $(whoami)@FPR_SERVER_HOST -p FPR_SSH_REMOTE_PORT\" and \"cd $(pwd)\""
+        mkdir -p  ~/.ssh/
+        echo $RSA_PUB >> ~/.ssh/authorized_keys
+        chmod 600 ~/.ssh/authorized_keys
+        wget https://github.com/fatedier/frp/releases/download/v0.32.1/frp_0.32.1_linux_amd64.tar.gz -O frp.tar.gz
+        tar xvzf frp.tar.gz -C /opt
+        mv /opt/frp* /opt/frp
+        /opt/frp/frpc tcp --server_addr $FPR_SERVER_ADDR --token $FPR_TOKEN  --local_port $SSH_PORT  --remote_port $FPR_SSH_REMOTE_PORT
+    - name: Test with pytest
+      run: |
+        export ALLOW_OPENAI_API_CALL=0
+        echo "${{ secrets.METAGPT_KEY_YAML }}" | base64 -d > config/key.yaml
+        mkdir -p ~/.metagpt && echo "${{ secrets.METAGPT_CONFIG2_YAML }}" | base64 -d > ~/.metagpt/config2.yaml
+        echo "${{ secrets.SPARK_YAML }}" | base64 -d > ~/.metagpt/spark.yaml
+        pytest tests/ --doctest-modules --cov=./metagpt/ --cov-report=xml:cov.xml --cov-report=html:htmlcov --durations=20 | tee unittest.txt
+    - name: Show coverage report
+      run: |
+        coverage report -m
+    - name: Show failed tests and overall summary
+      run: |
+        grep -E "FAILED tests|ERROR tests|[0-9]+ passed," unittest.txt
+        failed_count=$(grep -E "FAILED|ERROR" unittest.txt | wc -l)
+        if [[ "$failed_count" -gt 0 ]]; then
+          echo "$failed_count failed lines found! Task failed."
+          exit 1
+        fi
+    - name: Upload pytest test results
+      uses: actions/upload-artifact@v3
+      with:
+        name: pytest-results-${{ matrix.python-version }}
+        path: |
+          ./unittest.txt
+          ./htmlcov/
+          ./tests/data/rsp_cache_new.json
+        retention-days: 3
+      if: ${{ always() }}
+    - name: Upload coverage reports to Codecov
+      uses: codecov/codecov-action@v3
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+      if: ${{ always() }}

.github/workflows/unittest.yaml

@@ -1,16 +1,16 @@
 name: Unit Tests
 
 on:
-  workflow_dispatch:
   pull_request_target:
   push: 
     branches:
-      - '*-debugger'
+      - 'main'
+      - 'dev'
+      - '*-release'
 
 jobs:
   build:
     runs-on: ubuntu-latest
-    environment: unittest
     strategy:
       matrix:
         # python-version: ['3.9', '3.10', '3.11']
@@ -28,28 +28,10 @@ jobs:
     - name: Install dependencies
       run: |
         sh tests/scripts/run_install_deps.sh
-    - name: Run reverse proxy script for ssh service
-      if: contains(github.ref, '-debugger')
-      continue-on-error: true
-      env:
-        FPR_SERVER_ADDR: ${{ secrets.FPR_SERVER_ADDR }}
-        FPR_TOKEN: ${{ secrets.FPR_TOKEN }}
-        FPR_SSH_REMOTE_PORT: ${{ secrets.FPR_SSH_REMOTE_PORT }}
-        RSA_PUB: ${{ secrets.RSA_PUB }}
-        SSH_PORT: ${{ vars.SSH_PORT || '22'}}
-      run: |
-        echo "Run \"ssh $(whoami)@FPR_SERVER_HOST -p FPR_SSH_REMOTE_PORT\" and \"cd $(pwd)\""
-        mkdir -p  ~/.ssh/
-        echo $RSA_PUB >> ~/.ssh/authorized_keys
-        chmod 600 ~/.ssh/authorized_keys
-        wget https://github.com/fatedier/frp/releases/download/v0.32.1/frp_0.32.1_linux_amd64.tar.gz -O frp.tar.gz
-        tar xvzf frp.tar.gz -C /opt
-        mv /opt/frp* /opt/frp
-        /opt/frp/frpc tcp --server_addr $FPR_SERVER_ADDR --token $FPR_TOKEN  --local_port $SSH_PORT  --remote_port $FPR_SSH_REMOTE_PORT
     - name: Test with pytest
       run: |
         export ALLOW_OPENAI_API_CALL=0
-        echo "${{ secrets.METAGPT_KEY_YAML }}" | base64 -d > config/key.yaml
+        mkdir -p ~/.metagpt && cp tests/config2.yaml ~/.metagpt/config2.yaml && cp tests/spark.yaml ~/.metagpt/spark.yaml
         pytest tests/ --doctest-modules --cov=./metagpt/ --cov-report=xml:cov.xml --cov-report=html:htmlcov --durations=20 | tee unittest.txt
     - name: Show coverage report
       run: |

.gitignore

@@ -172,8 +172,10 @@ tests/metagpt/utils/file_repo_git
 *.png
 htmlcov
 htmlcov.*
+cov.xml
 *.dot
 *.pkl
+*.faiss
 *-structure.csv
 *-structure.json
-
+metagpt/tools/schemas

Dockerfile

@@ -8,7 +8,7 @@ RUN apt update &&\
 
 # Install Mermaid CLI globally
 ENV CHROME_BIN="/usr/bin/chromium" \
-    PUPPETEER_CONFIG="/app/metagpt/config/puppeteer-config.json"\
+    puppeteer_config="/app/metagpt/config/puppeteer-config.json"\
     PUPPETEER_SKIP_CHROMIUM_DOWNLOAD="true"
 RUN npm install -g @mermaid-js/mermaid-cli &&\
     npm cache clean --force

README.md

@@ -55,30 +55,21 @@
 
 ### Pip installation
 
+> Ensure that Python 3.9+ is installed on your system. You can check this by using: `python --version`.  
+> You can use conda like this: `conda create -n metagpt python=3.9 && conda activate metagpt`
+
 ```bash
-# Step 1: Ensure that Python 3.9+ is installed on your system. You can check this by using:
-# You can use conda to initialize a new python env
-#     conda create -n metagpt python=3.9
-#     conda activate metagpt
-python3 --version
-
-# Step 2: Clone the repository to your local machine for latest version, and install it.
-git clone https://github.com/geekan/MetaGPT.git
-cd MetaGPT
-pip3 install -e .     # or pip3 install metagpt  # for stable version
-
-# Step 3: setup your OPENAI_API_KEY, or make sure it existed in the env
-mkdir ~/.metagpt
-cp config/config.yaml ~/.metagpt/config.yaml
-vim ~/.metagpt/config.yaml
-
-# Step 4: run metagpt cli
-metagpt "Create a 2048 game in python"
-
-# Step 5 [Optional]: If you want to save the artifacts like diagrams such as quadrant chart, system designs, sequence flow in the workspace, you can execute the step before Step 3. By default, the framework is compatible, and the entire process can be run completely without executing this step.
-# If executing, ensure that NPM is installed on your system. Then install mermaid-js. (If you don't have npm in your computer, please go to the Node.js official website to install Node.js https://nodejs.org/ and then you will have npm tool in your computer.)
-npm --version
-sudo npm install -g @mermaid-js/mermaid-cli
+pip install metagpt
+metagpt --init-config  # create ~/.metagpt/config2.yaml, modify it to your own config
+metagpt "Create a 2048 game"  # this will create a repo in ./workspace
+```
+
+or you can use it as library
+
+```python
+from metagpt.software_company import generate_repo, ProjectRepo
+repo: ProjectRepo = generate_repo("Create a 2048 game")  # or ProjectRepo("<path>")
+print(repo)  # it will print the repo structure with files
 ```
 
 detail installation please refer to [cli_install](https://docs.deepwisdom.ai/main/en/guide/get_started/installation.html#install-stable-version)
@@ -87,19 +78,19 @@ detail installation please refer to [cli_install](https://docs.deepwisdom.ai/mai
 > Note: In the Windows, you need to replace "/opt/metagpt" with a directory that Docker has permission to create, such as "D:\Users\x\metagpt"
 
 ```bash
-# Step 1: Download metagpt official image and prepare config.yaml
+# Step 1: Download metagpt official image and prepare config2.yaml
 docker pull metagpt/metagpt:latest
 mkdir -p /opt/metagpt/{config,workspace}
-docker run --rm metagpt/metagpt:latest cat /app/metagpt/config/config.yaml > /opt/metagpt/config/key.yaml
-vim /opt/metagpt/config/key.yaml # Change the config
+docker run --rm metagpt/metagpt:latest cat /app/metagpt/config/config2.yaml > /opt/metagpt/config/config2.yaml
+vim /opt/metagpt/config/config2.yaml # Change the config
 
 # Step 2: Run metagpt demo with container
 docker run --rm \
     --privileged \
-    -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \
+    -v /opt/metagpt/config/config2.yaml:/app/metagpt/config/config2.yaml \
     -v /opt/metagpt/workspace:/app/metagpt/workspace \
     metagpt/metagpt:latest \
-    metagpt "Write a cli snake game"
+    metagpt "Create a 2048 game"
 ```
 
 detail installation please refer to [docker_install](https://docs.deepwisdom.ai/main/en/guide/get_started/installation.html#install-with-docker)

config/config2.yaml

@@ -0,0 +1,3 @@
+llm:
+  api_key: "YOUR_API_KEY"
+  model: "gpt-4-turbo-preview"  # or gpt-3.5-turbo-1106 / gpt-4-1106-preview