Github Actions refactoring to use Poetry/TestContainers

.github/workflows/lint.yml

@@ -21,7 +21,7 @@ jobs:
         # Starting new jobs is also relatively slow,
         # so linting on fewer versions makes CI faster.
         python-version:
-          - "3.8"
+          - "3.9"
           - "3.11"
 
     steps:
@@ -30,19 +30,17 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install .[dev,all]
-
+        poetry install
     - name: check-sort-import
       run: |
-        make check-sort-imports
-
+        poetry run check-sort-imports
     - name: check-black-format
       run: |
-        make check-format
-
-    - name: check-mypy
-      run: |
-        make mypy
+        poetry run check-format
+    # - name: check-mypy
+    #   run: |
+    #     poetry run mypy

.github/workflows/run_tests.yml

@@ -15,9 +15,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8, 3.9, '3.10', 3.11]
+        python-version: [3.9, '3.10', 3.11]
         connection: ['hiredis', 'plain']
-        redis-stack-version: ['6.2.6-v9', 'latest']
+        redis-stack-version: ['6.2.6-v9', 'latest', 'edge']
 
     services:
       redis:
@@ -33,20 +33,21 @@ jobs:
         python-version: ${{ matrix.python-version }}
         cache: 'pip'
 
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install .[dev,all]
+        poetry install
 
     - name: Install hiredis if needed
       if: matrix.connection == 'hiredis'
       run: |
-        pip install hiredis
+        poetry add hiredis
 
-    - name: Start Redis
+    - name: Set Redis version
       run: |
-        REDIS_URL=redis://localhost:6379
-        echo REDIS_URL=$REDIS_URL >> $GITHUB_ENV
+        echo "REDIS_VERSION=${{ matrix.redis-stack-version }}" >> $GITHUB_ENV
 
     - name: Authenticate to Google Cloud
       uses: google-github-actions/auth@v1
@@ -61,12 +62,12 @@ jobs:
         GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
         COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
       run: |
-        make test-cov
+        poetry run test-cov
 
     - name: Run tests
       if: matrix.connection != 'plain' || matrix.redis-stack-version != 'latest'
       run: |
-        SKIP_VECTORIZERS=True make test-cov
+        SKIP_VECTORIZERS=True poetry run test-cov
 
     - name: Run notebooks
       if: matrix.connection == 'plain' && matrix.redis-stack-version == 'latest'
@@ -76,10 +77,11 @@ jobs:
         GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
         COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
       run: |
-        cd docs/ && treon -v --exclude="./examples/openai_qna.ipynb"
+        cd docs/ && poetry run treon -v --exclude="./examples/openai_qna.ipynb"
 
     - name: Publish coverage results
-      uses: codecov/codecov-action@v2
+      uses: codecov/codecov-action@v4
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
-        fail_ci_if_error: true
+        files: ./coverage.xml
+        fail_ci_if_error: true

.github/workflows/test-publish.yml

@@ -11,20 +11,22 @@ jobs:
   build_release:
     name: Build Release
     runs-on: ubuntu-latest
+
     steps:
       - uses: actions/checkout@v3
+
       - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
           python-version: "3.10"
 
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install setuptools wheel
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
 
-      - name: create python package
-        run: python setup.py sdist bdist_wheel
+      - name: Build package
+        run: |
+          poetry version ${{ github.event.inputs.version }}
+          poetry build
 
       - name: Create Release
         uses: ncipollo/release-action@v1
@@ -36,9 +38,9 @@ jobs:
           tag: ${{ github.event.inputs.version }}
           commit: main
 
-      - name: deploy
+      - name: Publish to TestPyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           user: __token__
-          password: ${{ secrets.PYPI }}
+          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
           repository_url: https://test.pypi.org/legacy/

.gitignore

@@ -8,4 +8,5 @@ scratch
 *.csv
 wiki_schema.yaml
 docs/_build/
-.venv
+.venv
+coverage.xml

conftest.py

@@ -12,7 +12,6 @@ def redis_container():
 
     redis_host, redis_port = compose.get_service_host_and_port("redis", 6379)
     redis_url = f"redis://{redis_host}:{redis_port}"
-    print(f"REDIS TESTCONTAINERS => {redis_url}")
     os.environ["REDIS_URL"] = redis_url
 
     yield compose