Merge refactoring changes

.github/workflows/build.yml

@@ -0,0 +1,41 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Test Python package
+
+on:
+  push:
+    branches: [ "main", "release" ]
+  pull_request:
+    branches: [ "main", "release" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        python setup.py develop
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Test with pytest
+      run: |
+        python -m pytest

.github/workflows/run_experiment.yml

@@ -4,17 +4,12 @@ on:
   workflow_dispatch:
     inputs:
       model:
-        description: "Model to use for fixing (gpt-3.5, gpt-4)"
-        type: string
-        default: "gpt-3.5"
-      useModelOnly:
-          description: "Run experiment with no external sources"
-          type: boolean
-          default: false
-      useDoc:
-        description: "Run experiment with references from Documentation/release notes"
-        type: boolean
-        default: true
+        description: "Model to use for fixing (gpt-3.5-turbo-0125, gpt-4)"
+        type: choice
+        options:
+          - "gpt-3.5-turbo-0125"
+          - "gpt-4"
+        default: "gpt-3.5-turbo-0125"
       compareTo:
         description: "Run number of previous run to compare to (leave empty to skip comparison)"
         default: ""
@@ -31,13 +26,11 @@ jobs:
     outputs:
       libraries: "${{ steps.parse_libraries.outputs.libraries }}"
       model: "${{ github.event.inputs.model }}"
-      useModelOnly: "${{ github.event.inputs.useModelOnly || false }}"
-      useDoc: "${{ github.event.inputs.useDoc || true }}"
       threshold: "${{ github.event.inputs.simthreshold || 0 }}"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.10'
 
@@ -62,14 +55,14 @@ jobs:
         library: ${{ fromJson(needs.setup.outputs.libraries) }}
     steps:      
       - name: Checkout github repo (+ download lfs dependencies)
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           lfs: true
 
       - name: Pull LFS objects
         run: git lfs pull
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.10'
 
@@ -91,41 +84,17 @@ jobs:
         uses: mxschmitt/action-tmate@v3
         if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
 
-      - name: Run example update with no sources
-        if: ${{ github.event.inputs.useModelOnly == 'true' }}
+      - name: Run upgraider on given lib
         env:
           OPENAI_API_KEY: "${{ secrets.OPENAI_API_KEY }}"
           OPENAI_ORG: "${{ secrets.OPENAI_ORG }}"
-          GPT4_ENDPOINT: ${{ secrets.GPT4_ENDPOINT }}
-          GPT4_AUTH_HEADERS: ${{ secrets.GPT4_AUTH_HEADERS }}
         run: |
           library_name=${{ matrix.library.name }}
           curr_dir=`pwd`
-          outputdir="$curr_dir/results/$library_name/modelonly"
-          mkdir -p $outputdir 
-          python src/upgraider/fix_lib_examples.py \
+          outputdir="$curr_dir/results/"
+          python src/upgraider/run_experiment.py \
             --libpath ${{ matrix.library.path }} \
             --outputDir $outputdir \
-            --dbsource modelonly \
-            --threshold ${{ needs.setup.outputs.threshold }} \
-            --model ${{ needs.setup.outputs.model }} \
-
-      - name: Run example update with doc sources
-        if: ${{ needs.setup.outputs.useDoc == 'true' }}
-        env:
-          OPENAI_API_KEY: "${{ secrets.OPENAI_API_KEY }}"
-          OPENAI_ORG: "${{ secrets.OPENAI_ORG }}"
-          GPT4_ENDPOINT: ${{ secrets.GPT4_ENDPOINT }}
-          GPT4_AUTH_HEADERS: ${{ secrets.GPT4_AUTH_HEADERS }}
-        run: |
-          library_name=${{ matrix.library.name }}
-          curr_dir=`pwd`
-          outputdir="$curr_dir/results/$library_name/doc"
-          mkdir -p $outputdir 
-          python src/upgraider/fix_lib_examples.py \
-            --libpath ${{ matrix.library.path }} \
-            --outputDir $outputdir \
-            --dbsource doc \
             --threshold ${{ needs.setup.outputs.threshold }} \
             --model ${{ needs.setup.outputs.model }} \
 
@@ -134,7 +103,7 @@ jobs:
           zip -r results.zip results
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: results-${{ matrix.library.name }}
           path: "results.zip"
@@ -146,7 +115,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Download output zips
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
 
       - name: Combine output zips
         run: |
@@ -157,19 +126,19 @@ jobs:
           done
           zip -r results.zip results
       - name: Upload combined output files
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: results-all
           path: results.zip
-
+        
   generate-report:
     needs:
       - combine_output
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.10'
 
@@ -179,16 +148,16 @@ jobs:
           python setup.py develop
 
       - name: Download artifacts for this run
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: results-all
           path: results
 
       - name: Download artifacts for comparison run
         if: ${{ github.event.inputs.compareTo != '' }}
-        uses: dawidd6/action-download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
-          run_number: ${{ github.event.inputs.compareTo }}
+          run-id: ${{ github.event.inputs.compareTo }}
           name: results-all
           path: baseline
 

README.md

@@ -13,7 +13,7 @@ Note that UpgrAIder represents an early exploration of the above technique, and
 ```
 python -m venv .venv
 source .venv/bin/activate
-pip install -r requirements
+pip install -r requirements.txt
 python setup.py develop
 ```
 
@@ -26,7 +26,7 @@ python setup.py develop
 	cat > .env <<EOL
 	OPENAI_API_KEY=...
 	OPENAI_ORG=...
-	SCRATCH_VENV=<path to a folder that already has a venv we can activate>
+	SCRATCH_VENV=<absolute path to a folder that already has a venv we can activate>
 	```
 
 ## Running
@@ -43,15 +43,17 @@ The above script looks for sections with certain keywords related to APIs and/or
 
 ### Updating a single code example
 
-`src/upgraider/fix_code_examples.py` is the file responsible for this. Run `python upgraider/fix_lib_examples.py --help` to see the required command lines. To run a single example, make sure to specify `--examplefile`; otherwise, it will run on all the examples available for that library.
+`src/upgraider/fix_code_examples.py` is the file responsible for this. Run `python src/upgraider/fix_lib_examples.py --help` to see the required command lines. To run a single example, make sure to specify `--examplefile`; otherwise, it will run on all the examples available for that library.
 
 ### Running a full experiment
 
-Run `python src/upgraider/run_experiment.py`. This will attempt to run upgraider on *all* code examples avaiable for *all* libraries in the `libraries` folder. The output data and reports will be written to the `output` folder.
+Run `python src/upgraider/run_experiment.py --outputDir <absolute path of output folder>` This will attempt to run upgraider on *all* code examples avaiable for *all* libraries in the `libraries` folder. The output data and reports will be written to `outputDir`. 
 
-### Using Actions to run experiments
+To create a markdown report summarizing the results, use the `src/benchmark/parse_results.py` script while passing the output directory you wrote results to above. For example `python src/benchmark/parse_reports.py --outputdir output/`.
 
-The `run_experiment` workflow allows you to run a full experiment on the available libraries. It produces a markdown report of the results. Note that you need to set the required environment variables (i.e., API keys etc) as repository secrets.
+### Using GitHub Actions to run experiments
+
+The `run_experiment` workflow allows you to run a full experiment on the available libraries. It produces a markdown report of the results. Note that you need to configure your repository with two repository secrets `OPENAI_API_KEY` and `OPENAI_ORG`.
 
 ### Running Tests
 

libraries/numpy/library.json

@@ -1,6 +1,6 @@
 {
     "name": "numpy",
     "ghurl": "https://github.com/numpy/numpy",
-    "baseversion": "v1.22.3",
-    "currentversion": "v1.24.2"
+    "baseversion": "v1.24",
+    "currentversion": "v2.0"
 }

libraries/pandas/requirements.txt

@@ -1 +1,2 @@
 openpyxl
+numpy==1.26.4

requirements.txt

@@ -10,3 +10,5 @@ stackapi
 tiktoken
 dataclasses_json
 pytest
+docutils==0.21.1
+setuptools==69.5.1