removed snowflake and redshift from integration testing (#982)

.github/workflows/ci-integration-db-sqlalchemy-v1.yaml

@@ -0,0 +1,46 @@
+# CI - DB Integration - SqlAlchemy v1 is designed to run integration testing against SQLAlchemy v1.
+# Target database: Clickhouse
+# Sqlalchemy version: 1.4 (required by clickhouse-sqlalchemy)
+
+name: CI - DB Integration - SQLAlchemy V1
+
+on:
+  workflow_call:
+
+jobs:
+  database-sqlalchemy-v1-integration-test:
+    strategy:
+      matrix:
+        python-version: ['3.11']
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+
+    env:
+      PLOOMBER_VERSION_CHECK_DISABLED: true
+      PYTHON_VERSION: ${{ matrix.python-version }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade nox
+          nox --session test_integration_sqlachemy_v1 --install-only
+
+      - name: Integration Test
+        run: |
+          nox --session test_integration_sqlachemy_v1 --no-install --reuse-existing-virtualenvs
+
+      - name: Upload failed images artifacts
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+            name: failed-image-artifacts-sqlalchemy-v1 ${{ matrix.os }} ${{ matrix.python-version }}
+            path: result_images/

.github/workflows/ci.yaml

@@ -71,12 +71,11 @@ jobs:
             exit 0
           fi
 
-      
-  integration-test-live:
+
+  integration-test-sqlalchemy-v1:
     needs: [preliminary]
     if: needs.preliminary.outputs.check_doc_modified == 'failure'
-    uses: ./.github/workflows/ci-integration-db-live.yaml
-    secrets: inherit
+    uses: ./.github/workflows/ci-integration-db-sqlalchemy-v1.yaml
 
   integration-test-non-live:
     needs: [preliminary]
@@ -197,7 +196,7 @@ jobs:
 
 
   release:
-    needs: [test, test-sqlalchemy-v1, check, integration-test-non-live, integration-test-live]
+    needs: [test, test-sqlalchemy-v1, check, integration-test-non-live, integration-test-sqlalchemy-v1]
     if: startsWith(github.ref, 'refs/tags') && github.event_name != 'pull_request'
     runs-on: ubuntu-latest
 

doc/community/developer-guide.md

@@ -473,13 +473,38 @@ To see the databases available, check out [`src/tests/integration/conftest.py`](
 
 ### Integration tests with cloud databases
 
-We run integration tests against cloud databases like Snowflake, which requires using pre-registered accounts to evaluate their behavior. To initiate these tests, please create a branch in our [ploomber/jupyter repository](https://github.com/ploomber/jupysql).
+Currently, we do not run integration tests against cloud databases like Snowflake and Amazon Redshift.
 
-Please note that if you submit a pull request from a forked repository, the integration testing phase will be skipped because the pre-registered accounts won't be accessible.
+To run Snowflake integration tests locally first set your Snowflake account's username and password:
+
+```bash
+export SF_USERNAME="username"
+export SF_PASSWORD="password"
+```
+
+Then run the pytest command:
+
+```bash
+pytest src/tests/integration -k snowflake
+```
+
+Similarly, for Redshift, set the following environment variables:
+
+```bash
+export REDSHIFT_USERNAME="username"
+export REDSHIFT_PASSWORD="password"
+export REDSHIFT_HOST="host"
+```
+
+Then run the below command:
+
+```bash
+pytest src/tests/integration -k redshift
+```
 
 #### Using Snowflake
 
-While running live integration tests or testing manually with Snowflake, you may run into the below error:
+While testing manually with Snowflake, you may run into the below error:
 
 ```
 No active warehouse selected in the current session. Select an active warehouse with the 'use warehouse' command.

noxfile.py

@@ -127,6 +127,9 @@ def test_integration_cloud(session):
     Run integration tests on cloud databases (currently snowflake and redshift)
     (NOTE: the sqlalchemy-snowflake and sqlalchemy-redshift driver only work with
     SQLAlchemy 1.x)
+    This is disabled currently, refer: https://github.com/ploomber/jupysql/issues/984
+    If it is required to enable these tests add a job in
+    .github/workflows/ci.yaml file.
     """
 
     # TODO: do not require integration test dependencies if only running snowflake
@@ -147,6 +150,34 @@ def test_integration_cloud(session):
     )
 
 
+@nox.session(
+    venv_backend=VENV_BACKEND,
+    python=environ.get("PYTHON_VERSION", "3.11"),
+)
+def test_integration_sqlachemy_v1(session):
+    """
+    Run integration tests on SQLAlchemy v1
+    (NOTE: the clickhouse-sqlalchemy driver only works with
+    SQLAlchemy 1.x)
+    """
+
+    # tests
+    _install(session, integration=True)
+    session.install(
+        "snowflake-sqlalchemy",
+        "redshift-connector",
+        "sqlalchemy-redshift",
+        "clickhouse-sqlalchemy",
+    )
+    session.run(
+        "pytest",
+        "src/tests/integration",
+        "-k",
+        "clickhouse",
+        "-v",
+    )
+
+
 @nox.session(
     venv_backend=VENV_BACKEND,
     python=environ.get("PYTHON_VERSION", "3.11"),

setup.py

@@ -32,7 +32,8 @@
 DEV = [
     "flake8",
     "pytest",
-    "pandas",  # previously pinned to 2.0.3
+    # 24/01/24 Pandas 2.2.0 breaking CI: https://github.com/ploomber/jupysql/issues/983
+    "pandas<2.2.0",  # previously pinned to 2.0.3
     "polars==0.17.2",  # 04/18/23 this breaks our CI
     "pyarrow",
     "invoke",

src/tests/integration/test_generic_db_operations.py

@@ -24,6 +24,10 @@
     "ip_with_spark",
 ]
 
+# NOTE: We don't need to add tests for Snowflake and Redshift
+# for future PRs.
+# Reference issue: https://github.com/ploomber/jupysql/issues/984
+
 
 @pytest.fixture(autouse=True)
 def run_around_tests(tmpdir_factory):