Refactor nox to separate integration tests (#2950)

* Split tests on nox * Fix marker * Improve how we skip tests * Make dependencies not required * More ignores and import fixes * More import skips * Remove aiohttp dep in tests * Fix * Remove more imports * More imports * One more import * More skips * Older pydantic when using starlite * Missing dep * Missing dep * Missing deps * cli tests * Fix federation * TMP disable cache * Reskip * Reenable ws tests * Fix more tests * Add coverage for tests * Fix import path * Fix test * Fix markers * Test fix * Fix pydantic tests * Remove unused fixtures * Run windows with poetry * Remove use * Fix command
strawberry-graphql · Jul 16, 2023 · 5c0d775 · 5c0d775
1 parent 60bfd76
commit 5c0d775
Show file tree

Hide file tree

Showing 46 changed files with 809 additions and 702 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -36,7 +36,7 @@ ignore_errors = True
 
 omit =
   ./.venv/**
-  ./tests/*
+  noxfile.py
 
 [html]
 directory = coverage_html_report
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -78,6 +78,7 @@ jobs:
       - name: coverage xml
         run: coverage xml -i
         if: ${{ always() }}
+
       - uses: codecov/codecov-action@v3
         if: ${{ always() }}
         with:
@@ -148,25 +149,29 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-      - uses: wntrblm/nox@main
+      - run: pipx install poetry
+      - run: pipx install coverage
+      - uses: actions/setup-python@v4
+        id: setup-python
         with:
-          python-versions: "3.7, 3.8, 3.9, 3.10, 3.11"
+          python-version: "3.11"
+          cache: "poetry"
 
-      - name: Pip and nox cache
-        id: cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache
-            ~/.nox
-            .nox
-          key:
-            ${{ runner.os }}-nox-windows-${{ hashFiles('**/poetry.lock') }}-${{
-            hashFiles('**/noxfile.py') }}
-          restore-keys: |
-            ${{ runner.os }}-nox-windows
-            ${{ runner.os }}-nox-
+      - run: poetry install --with integrations
+        if: steps.setup-python.outputs.cache-hit != 'true'
 
-      - run: pipx install poetry
-      - run: pipx inject nox nox-poetry
-      - run: nox -r -s "Tests" -p 3.11
+      # we use poetry directly instead of nox since we want to
+      # test all integrations at once on windows
+      - run: |
+          poetry run pytest --cov=. --cov-append --cov-report=xml -n auto --showlocals -vv
+
+      - name: coverage xml
+        run: coverage xml -i
+        if: ${{ always() }}
+
+      - uses: codecov/codecov-action@v3
+        if: ${{ always() }}
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          fail_ci_if_error: true
+          verbose: true
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -117,7 +117,7 @@ Now, you will need to install the required dependencies for Strawberry and be su
 that the current tests are passing on your machine:
 
 ```shell
-$ poetry install
+$ poetry install --with integrations
 $ poetry run pytest tests -n auto
 $ poetry run mypy
 ```

diff --git a/federation-compatibility/Dockerfile b/federation-compatibility/Dockerfile
@@ -9,7 +9,7 @@ COPY pyproject.toml ./
 COPY poetry.lock ./
 COPY README.md ./
 
-RUN poetry install
+RUN poetry install --with integrations
 
 COPY federation-compatibility/schema.py ./
 

diff --git a/noxfile.py b/noxfile.py
@@ -1,32 +1,55 @@
 import nox
 from nox_poetry import Session, session
 
+nox.options.reuse_existing_virtualenvs = True
+nox.options.error_on_external_run = True
+
 PYTHON_VERSIONS = ["3.11", "3.10", "3.9", "3.8", "3.7"]
 
 
+COMMON_PYTEST_OPTIONS = [
+    "--cov=.",
+    "--cov-append",
+    "--cov-report=xml",
+    "-n",
+    "auto",
+    "--showlocals",
+    "-vv",
+    "--ignore=tests/mypy",
+    "--ignore=tests/pyright",
+    "--ignore=tests/cli",
+    # TODO: reintroduce this in its own test session
+    "--ignore=tests/experimental/pydantic",
+]
+
+INTEGRATIONS = [
+    "asgi",
+    "aiohttp",
+    "chalice",
+    "channels",
+    "django",
+    "fastapi",
+    "flask",
+    "sanic",
+    "starlite",
+    "pydantic",
+]
+
+
 @session(python=PYTHON_VERSIONS, name="Tests", tags=["tests"])
 def tests(session: Session) -> None:
     session.run_always("poetry", "install", external=True)
 
+    markers = (
+        ["-m", f"not {integration}", f"--ignore=tests/{integration}"]
+        for integration in INTEGRATIONS
+    )
+    markers = [item for sublist in markers for item in sublist]
+
     session.run(
         "pytest",
-        "--cov=strawberry",
-        "--cov-append",
-        "--cov-report=xml",
-        "-n",
-        "auto",
-        "--showlocals",
-        "-vv",
-        "-m",
-        "not starlette",
-        "-m",
-        "not django",
-        "-m",
-        "not starlite",
-        "-m",
-        "not pydantic",
-        "--ignore=tests/mypy",
-        "--ignore=tests/pyright",
+        *COMMON_PYTEST_OPTIONS,
+        *markers,
     )
 
 
@@ -36,19 +59,9 @@ def tests_django(session: Session, django: str) -> None:
     session.run_always("poetry", "install", external=True)
 
     session._session.install(f"django~={django}")  # type: ignore
+    session._session.install("pytest-django")  # type: ignore
 
-    session.run(
-        "pytest",
-        "--cov=strawberry",
-        "--cov-append",
-        "--cov-report=xml",
-        "-n",
-        "auto",
-        "--showlocals",
-        "-vv",
-        "-m",
-        "django",
-    )
+    session.run("pytest", *COMMON_PYTEST_OPTIONS, "-m", "django")
 
 
 @session(python=["3.11"], name="Starlette tests", tags=["tests"])
@@ -58,36 +71,38 @@ def tests_starlette(session: Session, starlette: str) -> None:
 
     session._session.install(f"starlette=={starlette}")  # type: ignore
 
-    session.run(
-        "pytest",
-        "--cov=strawberry",
-        "--cov-append",
-        "--cov-report=xml",
-        "-n",
-        "auto",
-        "--showlocals",
-        "-vv",
-        "-m",
-        "starlette",
-    )
+    session.run("pytest", *COMMON_PYTEST_OPTIONS, "-m", "asgi")
 
 
-@session(python=["3.11"], name="Litestar tests", tags=["tests"])
-def tests_litestar(session: Session) -> None:
+@session(python=["3.11"], name="Test integrations", tags=["tests"])
+@nox.parametrize(
+    "integration",
+    [
+        "aiohttp",
+        "chalice",
+        "channels",
+        "fastapi",
+        "flask",
+        "sanic",
+        "starlite",
+    ],
+)
+def tests_integrations(session: Session, integration: str) -> None:
     session.run_always("poetry", "install", external=True)
 
-    session.run(
-        "pytest",
-        "--cov=strawberry",
-        "--cov-append",
-        "--cov-report=xml",
-        "-n",
-        "auto",
-        "--showlocals",
-        "-vv",
-        "-m",
-        "starlite",
-    )
+    session._session.install(integration)  # type: ignore
+
+    if integration == "aiohttp":
+        session._session.install("pytest-aiohttp")  # type: ignore
+    elif integration == "flask":
+        session._session.install("pytest-flask")  # type: ignore
+    elif integration == "channels":
+        session._session.install("pytest-django")  # type: ignore
+        session._session.install("daphne")  # type: ignore
+    elif integration == "starlite":
+        session._session.install("pydantic<2.0")  # type: ignore
+
+    session.run("pytest", *COMMON_PYTEST_OPTIONS, "-m", integration)
 
 
 @session(python=["3.11"], name="Pydantic tests", tags=["tests"])
@@ -100,25 +115,21 @@ def test_pydantic(session: Session, pydantic: str) -> None:
 
     session.run(
         "pytest",
-        "--cov=strawberry",
+        "--cov=.",
         "--cov-append",
         "--cov-report=xml",
-        "-n",
-        "auto",
-        "--showlocals",
-        "-vv",
         "-m",
         "pydantic",
     )
 
 
 @session(python=PYTHON_VERSIONS, name="Mypy tests")
 def tests_mypy(session: Session) -> None:
-    session.run_always("poetry", "install", external=True)
+    session.run_always("poetry", "install", "--with", "integrations", external=True)
 
     session.run(
         "pytest",
-        "--cov=strawberry",
+        "--cov=.",
         "--cov-append",
         "--cov-report=xml",
         "tests/mypy",
@@ -133,7 +144,7 @@ def tests_pyright(session: Session) -> None:
 
     session.run(
         "pytest",
-        "--cov=strawberry",
+        "--cov=.",
         "--cov-append",
         "--cov-report=xml",
         "tests/pyright",
@@ -143,6 +154,23 @@ def tests_pyright(session: Session) -> None:
 
 @session(name="Mypy", tags=["lint"])
 def mypy(session: Session) -> None:
-    session.run_always("poetry", "install", external=True)
+    session.run_always("poetry", "install", "--with", "integrations", external=True)
 
     session.run("mypy", "--config-file", "mypy.ini")
+
+
+@session(python=PYTHON_VERSIONS, name="CLI tests", tags=["tests"])
+def tests_cli(session: Session) -> None:
+    session.run_always("poetry", "install", external=True)
+
+    session._session.install("uvicorn")  # type: ignore
+    session._session.install("starlette")  # type: ignore
+
+    session.run(
+        "pytest",
+        "--cov=.",
+        "--cov-append",
+        "--cov-report=xml",
+        "tests/cli",
+        "-vv",
+    )