From f43a570f8295b7525aaac4f9b4fe8550efd3cd7c Mon Sep 17 00:00:00 2001 From: Eduardo Blancas Date: Mon, 24 Jul 2023 13:05:17 -0600 Subject: [PATCH] migrates tests to use nox --- .github/workflows/ci-integration-db-live.yaml | 12 ++++-- .github/workflows/ci-integration-db.yaml | 12 +++--- .github/workflows/ci.yaml | 28 +++++-------- doc/community/developer-guide.md | 6 +-- noxfile.py | 41 ++++++++++++++----- 5 files changed, 59 insertions(+), 40 deletions(-) diff --git a/.github/workflows/ci-integration-db-live.yaml b/.github/workflows/ci-integration-db-live.yaml index fe19cbd04..88d3a8b15 100644 --- a/.github/workflows/ci-integration-db-live.yaml +++ b/.github/workflows/ci-integration-db-live.yaml @@ -18,6 +18,7 @@ jobs: env: PLOOMBER_VERSION_CHECK_DISABLED: true + PYTHON_VERSION: ${{ matrix.python-version }} steps: - name: Checkout @@ -44,7 +45,11 @@ jobs: - name: Install dependencies if: ${{ steps.secret-check.outputs.available == 'true' }} run: - pip install ".[integration]" + + python -m pip install --upgrade pip + python -m pip install --upgrade nox + nox --session test_integration_snowflake --install-only + - name: Integration Test if: ${{ steps.secret-check.outputs.available == 'true'}} env: @@ -52,9 +57,8 @@ jobs: SF_PASSWORD: ${{ secrets.SF_PASSWORD }} SF_DATABASE: ${{ secrets.SF_DATABASE }} run: | - # Run the integration test with live option - python -c "import sqlalchemy; print('Sqlalchemy version ', sqlalchemy.__version__)" - pytest src/tests/integration --verbose --live + + nox --session test_integration_snowflake --no-install --reuse-existing-virtualenvs - name: Upload failed images artifacts uses: actions/upload-artifact@v3 diff --git a/.github/workflows/ci-integration-db.yaml b/.github/workflows/ci-integration-db.yaml index 2ac20a03c..cb83c6ead 100644 --- a/.github/workflows/ci-integration-db.yaml +++ b/.github/workflows/ci-integration-db.yaml @@ -15,6 +15,7 @@ jobs: env: PLOOMBER_VERSION_CHECK_DISABLED: true + PYTHON_VERSION: ${{ matrix.python-version }} steps: @@ -37,13 +38,14 @@ jobs: echo 'export PATH="$PATH:/opt/mssql-tools18/bin"' >> ~/.bashrc source ~/.bashrc - pip install ".[integration]" - pip install "sqlalchemy>=2" + python -m pip install --upgrade pip + python -m pip install --upgrade nox + nox --session test_integration --install-only + - name: Integration Test run: | - # Run the integration test by pytest marker - python -c "import sqlalchemy; print('Sqlalchemy version ', sqlalchemy.__version__)" - pytest src/tests/integration --verbose + + nox --session test_integration --no-install --reuse-existing-virtualenvs - name: Upload failed images artifacts uses: actions/upload-artifact@v3 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6aa445d55..65ea5b762 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -77,6 +77,7 @@ jobs: if: needs.preliminary.outputs.check_doc_modified == 'failure' uses: ./.github/workflows/ci-integration-db-live.yaml secrets: inherit + integration-test-non-live: needs: [preliminary] if: needs.preliminary.outputs.check_doc_modified == 'failure' @@ -94,6 +95,7 @@ jobs: env: PLOOMBER_VERSION_CHECK_DISABLED: true + PYTHON_VERSION: ${{ matrix.python-version }} steps: @@ -108,25 +110,19 @@ jobs: - name: Lint run: | python -m pip install --upgrade pip - python -m pip install --upgrade pkgmt codespell + python -m pip install --upgrade pkgmt codespell nox pkgmt lint codespell - name: Install dependencies run: | - pip install . - # check package is importable - python -c "import sql" - pip install ".[dev]" - pip install sqlalchemy -U + nox --session test_unit --install-only - name: Test with pytest run: | - # ensure we're running sqlalchemy 2 - python -c "import sqlalchemy; assert int(sqlalchemy.__version__.split('.')[0]) == 2" - # run unit tests - pytest --durations-min=5 --ignore=src/tests/integration + + nox --session test_unit --no-install --reuse-existing-virtualenvs - name: Upload failed images artifacts uses: actions/upload-artifact@v3 @@ -147,6 +143,7 @@ jobs: env: PLOOMBER_VERSION_CHECK_DISABLED: true + PYTHON_VERSION: ${{ matrix.python-version }} steps: @@ -160,20 +157,17 @@ jobs: - name: Lint run: | - python -m pip install --upgrade pip pkgmt + python -m pip install --upgrade pip + python -m pip install --upgrade pkgmt nox pkgmt lint - name: Install dependencies run: | - pip install "sqlalchemy<2" - pip install ".[dev]" + nox --session test_unit_sqlalchemy_one --install-only - name: Test with pytest run: | - # ensure we're running sqlalchemy 1 - python -c "import sqlalchemy; assert int(sqlalchemy.__version__.split('.')[0]) == 1" - # run tests - pytest --durations-min=5 --ignore=src/tests/integration + nox --session test_unit_sqlalchemy_one --no-install --reuse-existing-virtualenvs - name: Upload failed images artifacts sqlalchemyv1 uses: actions/upload-artifact@v3 diff --git a/doc/community/developer-guide.md b/doc/community/developer-guide.md index 04a107fc9..2093048fe 100644 --- a/doc/community/developer-guide.md +++ b/doc/community/developer-guide.md @@ -110,7 +110,7 @@ In our codebase, we manage connections to databases with a `Connection` object, ### Working with connections -`Connection` should be exclusively used to manage database connections on the user's behalf and to obtain the current SQLAlchemy connection. We can access the current SQLAlchemy connection using `current.session`. +`ConnectionManager` should be exclusively used to manage database connections on the user's behalf and to obtain the current connection. We can access the current connection using `current`. ```{code-cell} ipython3 :tags: [remove-output] @@ -122,11 +122,11 @@ In our codebase, we manage connections to databases with a `Connection` object, ```{code-cell} ipython3 from sql.connection import ConnectionManager -conn = ConnectionManager.current.session +conn = ConnectionManager.current conn ``` -Functions that expect a `conn` (sometimes named `con`) input variable should only use SQLAlchemy connections. +Functions that expect a `conn` (sometimes named `con`) input variable should only use connections. ```python def histogram(payload, table, column, bins, with_=None, conn=None): diff --git a/noxfile.py b/noxfile.py index 7a4ad3497..544fea3f5 100644 --- a/noxfile.py +++ b/noxfile.py @@ -18,7 +18,7 @@ print("CONDA_PREFIX not found, creating envs in default location...") -DEV_ENV_NAME = "jupysql-integration" +DEV_ENV_NAME = "jupysql-env" INTEGRATION_CONDA_DEPENDENCIES = [ @@ -63,27 +63,37 @@ def _run_unit(session, skip_image_tests): args = [ "pytest", "src/tests/", - "--ignore src/tests/integration", + "--ignore", + "src/tests/integration", ] if skip_image_tests: args.extend( [ - "--ignore src/tests/test_ggplot.py", - "--ignore src/tests/test_magic_plot.py", + "--ignore", + "src/tests/test_ggplot.py", + "--ignore", + "src/tests/test_magic_plot.py", ] ) session.run(*args) -@nox.session(venv_backend="conda", name=DEV_ENV_NAME) +@nox.session( + venv_backend="conda", + name=DEV_ENV_NAME, + python=environ.get("PYTHON_VERSION", "3.11"), +) def setup(session): print("Installing requirements...") _install(session, integration=False) -@nox.session(venv_backend="conda") +@nox.session( + venv_backend="conda", + python=environ.get("PYTHON_VERSION", "3.11"), +) def test_unit(session): """Run unit tests (SQLAlchemy 2.x)""" SKIP_IMAGE_TEST = "--skip-image-tests" in session.posargs @@ -91,10 +101,13 @@ def test_unit(session): _install(session, integration=False) session.install("sqlalchemy>=2") _check_sqlalchemy(session, version=2) - _run_unit(skip_image_tests=SKIP_IMAGE_TEST) + _run_unit(session, skip_image_tests=SKIP_IMAGE_TEST) -@nox.session(venv_backend="conda") +@nox.session( + venv_backend="conda", + python=environ.get("PYTHON_VERSION", "3.11"), +) def test_unit_sqlalchemy_one(session): """Run unit tests (SQLAlchemy 1.x)""" SKIP_IMAGE_TEST = "--skip-image-tests" in session.posargs @@ -102,10 +115,13 @@ def test_unit_sqlalchemy_one(session): _install(session, integration=False) session.install("sqlalchemy<2") _check_sqlalchemy(session, version=1) - _run_unit(skip_image_tests=SKIP_IMAGE_TEST) + _run_unit(session, skip_image_tests=SKIP_IMAGE_TEST) -@nox.session(venv_backend="conda") +@nox.session( + venv_backend="conda", + python=environ.get("PYTHON_VERSION", "3.11"), +) def test_integration_snowflake(session): """ Run snowflake tests (NOTE: the sqlalchemy-snowflake driver only works with @@ -116,7 +132,10 @@ def test_integration_snowflake(session): session.run("pytest", "src/tests/integration", "-k", "snowflake") -@nox.session(venv_backend="conda") +@nox.session( + venv_backend="conda", + python=environ.get("PYTHON_VERSION", "3.11"), +) def test_integration(session): """Run integration tests (to check compatibility with databases)""" _install(session, integration=True)