Use VCR for tests (#375)

* uv add --dev inline-snapshot * uv add --dev pytest-recording * Configure vcr for openai requests * Apply VCR to all openai tests * Switch all tests to gpt-4o from 3.5-turbo * Add all openai cassettes * Add comment re _load_dotenv * Switch Anthropic to VCR * Add cassettes for Anthropic * Enable VCR for litellm_openai * Add cassettes for litellm_openai * Use VCR for litellm_anthropic * Use VCR for mistral * Add .env.template and use as default for tests * Enable ignoring VCR cassettes in github test workflow * mypy: add missing return type * Use VCR for ollama. Skip broken ollama tests. * Add VCR cassettes for litellm_ollama * Set asyncio_mode to auto, remove marks
jackmpcollins · Nov 17, 2024 · e08de50 · e08de50
1 parent cc0ac60
commit e08de50
Show file tree

Hide file tree

Showing 123 changed files with 15,321 additions and 104 deletions.
diff --git a/.env.template b/.env.template
@@ -0,0 +1,5 @@
+# To set API keys, copy this as .env and fill in the values
+# API keys are required to rewrite the VCR cassettes for the tests
+ANTHROPIC_API_KEY="anthropic_api_key"
+MISTRAL_API_KEY="mistral_api_key"
+OPENAI_API_KEY="openai_api_key"
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -7,6 +7,7 @@ on:
       - main
   workflow_dispatch:
     inputs:
+      # Options to test specific APIs ignoring the VCR cassettes
       anthropic:
         description: "Test using Anthropic API"
         type: boolean
@@ -48,22 +49,22 @@ jobs:
       - if: inputs.anthropic
         env:
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-        run: uv run pytest -vv tests/ -m 'anthropic'
+        run: uv run pytest -vv tests/ -m 'anthropic' --record-mode=rewrite
       - if: inputs.litellm_anthropic
         env:
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-        run: uv run pytest -vv tests/ -m 'litellm_anthropic'
+        run: uv run pytest -vv tests/ -m 'litellm_anthropic' --record-mode=rewrite
       - if: inputs.litellm_openai
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           OPENAI_ORG_ID: ${{ secrets.OPENAI_ORG_ID }}
-        run: uv run pytest -vv tests/ -m 'litellm_openai'
+        run: uv run pytest -vv tests/ -m 'litellm_openai' --record-mode=rewrite
       - if: inputs.mistral
         env:
           MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
-        run: uv run pytest -vv tests/ -m 'mistral'
+        run: uv run pytest -vv tests/ -m 'mistral' --record-mode=rewrite
       - if: inputs.openai
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           OPENAI_ORG_ID: ${{ secrets.OPENAI_ORG_ID }}
-        run: uv run pytest -vv tests/ -m 'openai'
+        run: uv run pytest -vv tests/ -m 'openai' --record-mode=rewrite
diff --git a/pyproject.toml b/pyproject.toml
@@ -22,6 +22,9 @@ build-backend = "hatchling.build"
 
 [dependency-groups]
 dev = [
+    "anthropic",
+    "inline-snapshot>=0.14.0",
+    "litellm",
     "logfire>=0.46.1,<0.50.0",
     "mypy",
     "pytest>=7.0.0",
@@ -30,9 +33,7 @@ dev = [
     "pytest-cov>=4.0.0",
     "python-dotenv>=1.0.1",
     "ruff>=0.3",
-    # optional-dependencies
-    "anthropic",
-    "litellm",
+    "pytest-recording>=0.13.2",
 ]
 docs = [
     "blacken-docs>=1.16.0",
@@ -64,7 +65,9 @@ venvPath = "."
 venv = ".venv"
 
 [tool.pytest.ini_options]
-addopts = "--cov=magentic --cov-report=term-missing -m 'not anthropic and not litellm_anthropic and not litellm_openai and not mistral and not litellm_ollama and not openai'"
+addopts = "--cov=magentic --cov-report=term-missing --block-network"
+asyncio_default_fixture_loop_scope = "function"
+asyncio_mode = "auto"
 markers = [
     "anthropic: Tests that query the Anthropic API. Requires the ANTHROPIC_API_KEY environment variable to be set.",
     "litellm_anthropic: Tests that query the Anthropic API via litellm. Requires the ANTHROPIC_API_KEY environment variable to be set.",

diff --git a/tests/cassettes/test_backend/test_openai_chat_model_completion.yaml b/tests/cassettes/test_backend/test_openai_chat_model_completion.yaml
@@ -0,0 +1,116 @@
+interactions:
+- request:
+    body: '{"messages": [{"role": "user", "content": "Hello!"}], "model": "gpt-4o",
+      "max_tokens": 5, "stream": true, "stream_options": {"include_usage": true},
+      "temperature": 0.5}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '168'
+      content-type:
+      - application/json
+      host:
+      - api.openai.com
+      user-agent:
+      - OpenAI/Python 1.54.4
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - 'false'
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.54.4
+      x-stainless-retry-count:
+      - '0'
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.10.15
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    body:
+      string: 'data: {"id":"chatcmpl-AU9UN6hG54O1PXRTx1nqzPVuMCrza","object":"chat.completion.chunk","created":1731749679,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_159d8341cc","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null}
+
+
+        data: {"id":"chatcmpl-AU9UN6hG54O1PXRTx1nqzPVuMCrza","object":"chat.completion.chunk","created":1731749679,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_159d8341cc","choices":[{"index":0,"delta":{"content":"Hello"},"logprobs":null,"finish_reason":null}],"usage":null}
+
+
+        data: {"id":"chatcmpl-AU9UN6hG54O1PXRTx1nqzPVuMCrza","object":"chat.completion.chunk","created":1731749679,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_159d8341cc","choices":[{"index":0,"delta":{"content":"!"},"logprobs":null,"finish_reason":null}],"usage":null}
+
+
+        data: {"id":"chatcmpl-AU9UN6hG54O1PXRTx1nqzPVuMCrza","object":"chat.completion.chunk","created":1731749679,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_159d8341cc","choices":[{"index":0,"delta":{"content":"
+        How"},"logprobs":null,"finish_reason":null}],"usage":null}
+
+
+        data: {"id":"chatcmpl-AU9UN6hG54O1PXRTx1nqzPVuMCrza","object":"chat.completion.chunk","created":1731749679,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_159d8341cc","choices":[{"index":0,"delta":{"content":"
+        can"},"logprobs":null,"finish_reason":null}],"usage":null}
+
+
+        data: {"id":"chatcmpl-AU9UN6hG54O1PXRTx1nqzPVuMCrza","object":"chat.completion.chunk","created":1731749679,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_159d8341cc","choices":[{"index":0,"delta":{"content":"
+        I"},"logprobs":null,"finish_reason":null}],"usage":null}
+
+
+        data: {"id":"chatcmpl-AU9UN6hG54O1PXRTx1nqzPVuMCrza","object":"chat.completion.chunk","created":1731749679,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_159d8341cc","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"length"}],"usage":null}
+
+
+        data: {"id":"chatcmpl-AU9UN6hG54O1PXRTx1nqzPVuMCrza","object":"chat.completion.chunk","created":1731749679,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_159d8341cc","choices":[],"usage":{"prompt_tokens":9,"completion_tokens":5,"total_tokens":14,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}}
+
+
+        data: [DONE]
+
+
+        '
+    headers:
+      CF-Cache-Status:
+      - DYNAMIC
+      CF-RAY:
+      - 8e367c85fb61cf0e-SJC
+      Connection:
+      - keep-alive
+      Content-Type:
+      - text/event-stream; charset=utf-8
+      Date:
+      - Sat, 16 Nov 2024 09:34:39 GMT
+      Server:
+      - cloudflare
+      Transfer-Encoding:
+      - chunked
+      X-Content-Type-Options:
+      - nosniff
+      access-control-expose-headers:
+      - X-Request-ID
+      alt-svc:
+      - h3=":443"; ma=86400
+      openai-processing-ms:
+      - '130'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      x-ratelimit-limit-requests:
+      - '500'
+      x-ratelimit-limit-tokens:
+      - '30000'
+      x-ratelimit-remaining-requests:
+      - '499'
+      x-ratelimit-remaining-tokens:
+      - '29992'
+      x-ratelimit-reset-requests:
+      - 120ms
+      x-ratelimit-reset-tokens:
+      - 16ms
+      x-request-id:
+      - req_6bcf128883d8d89f0477582d56a9a782
+    status:
+      code: 200
+      message: OK
+version: 1
diff --git a/tests/cassettes/test_chat/test_chat_asubmit.yaml b/tests/cassettes/test_chat/test_chat_asubmit.yaml
@@ -0,0 +1,130 @@
+interactions:
+- request:
+    body: '{"messages": [{"role": "user", "content": "Hello"}], "model": "gpt-4o",
+      "stream": true, "stream_options": {"include_usage": true}}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '130'
+      content-type:
+      - application/json
+      host:
+      - api.openai.com
+      user-agent:
+      - AsyncOpenAI/Python 1.54.4
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - async:asyncio
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.54.4
+      x-stainless-retry-count:
+      - '0'
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.10.15
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    body:
+      string: 'data: {"id":"chatcmpl-AU9UNy1IrwaLjfjH3NZALLZFyDgO4","object":"chat.completion.chunk","created":1731749679,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_7f6be3efb0","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null}
+
+
+        data: {"id":"chatcmpl-AU9UNy1IrwaLjfjH3NZALLZFyDgO4","object":"chat.completion.chunk","created":1731749679,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_7f6be3efb0","choices":[{"index":0,"delta":{"content":"Hello"},"logprobs":null,"finish_reason":null}],"usage":null}
+
+
+        data: {"id":"chatcmpl-AU9UNy1IrwaLjfjH3NZALLZFyDgO4","object":"chat.completion.chunk","created":1731749679,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_7f6be3efb0","choices":[{"index":0,"delta":{"content":"!"},"logprobs":null,"finish_reason":null}],"usage":null}
+
+
+        data: {"id":"chatcmpl-AU9UNy1IrwaLjfjH3NZALLZFyDgO4","object":"chat.completion.chunk","created":1731749679,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_7f6be3efb0","choices":[{"index":0,"delta":{"content":"
+        How"},"logprobs":null,"finish_reason":null}],"usage":null}
+
+
+        data: {"id":"chatcmpl-AU9UNy1IrwaLjfjH3NZALLZFyDgO4","object":"chat.completion.chunk","created":1731749679,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_7f6be3efb0","choices":[{"index":0,"delta":{"content":"
+        can"},"logprobs":null,"finish_reason":null}],"usage":null}
+
+
+        data: {"id":"chatcmpl-AU9UNy1IrwaLjfjH3NZALLZFyDgO4","object":"chat.completion.chunk","created":1731749679,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_7f6be3efb0","choices":[{"index":0,"delta":{"content":"
+        I"},"logprobs":null,"finish_reason":null}],"usage":null}
+
+
+        data: {"id":"chatcmpl-AU9UNy1IrwaLjfjH3NZALLZFyDgO4","object":"chat.completion.chunk","created":1731749679,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_7f6be3efb0","choices":[{"index":0,"delta":{"content":"
+        assist"},"logprobs":null,"finish_reason":null}],"usage":null}
+
+
+        data: {"id":"chatcmpl-AU9UNy1IrwaLjfjH3NZALLZFyDgO4","object":"chat.completion.chunk","created":1731749679,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_7f6be3efb0","choices":[{"index":0,"delta":{"content":"
+        you"},"logprobs":null,"finish_reason":null}],"usage":null}
+
+
+        data: {"id":"chatcmpl-AU9UNy1IrwaLjfjH3NZALLZFyDgO4","object":"chat.completion.chunk","created":1731749679,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_7f6be3efb0","choices":[{"index":0,"delta":{"content":"
+        today"},"logprobs":null,"finish_reason":null}],"usage":null}
+
+
+        data: {"id":"chatcmpl-AU9UNy1IrwaLjfjH3NZALLZFyDgO4","object":"chat.completion.chunk","created":1731749679,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_7f6be3efb0","choices":[{"index":0,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}],"usage":null}
+
+
+        data: {"id":"chatcmpl-AU9UNy1IrwaLjfjH3NZALLZFyDgO4","object":"chat.completion.chunk","created":1731749679,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_7f6be3efb0","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null}
+
+
+        data: {"id":"chatcmpl-AU9UNy1IrwaLjfjH3NZALLZFyDgO4","object":"chat.completion.chunk","created":1731749679,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_7f6be3efb0","choices":[],"usage":{"prompt_tokens":8,"completion_tokens":9,"total_tokens":17,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}}
+
+
+        data: [DONE]
+
+
+        '
+    headers:
+      CF-Cache-Status:
+      - DYNAMIC
+      CF-RAY:
+      - 8e367c8abcaced3c-SJC
+      Connection:
+      - keep-alive
+      Content-Type:
+      - text/event-stream; charset=utf-8
+      Date:
+      - Sat, 16 Nov 2024 09:34:40 GMT
+      Server:
+      - cloudflare
+      Transfer-Encoding:
+      - chunked
+      X-Content-Type-Options:
+      - nosniff
+      access-control-expose-headers:
+      - X-Request-ID
+      alt-svc:
+      - h3=":443"; ma=86400
+      openai-processing-ms:
+      - '223'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      x-ratelimit-limit-requests:
+      - '500'
+      x-ratelimit-limit-tokens:
+      - '30000'
+      x-ratelimit-remaining-requests:
+      - '499'
+      x-ratelimit-remaining-tokens:
+      - '29981'
+      x-ratelimit-reset-requests:
+      - 120ms
+      x-ratelimit-reset-tokens:
+      - 38ms
+      x-request-id:
+      - req_15139d22fd6669e7b4f8198a4c83fa32
+    status:
+      code: 200
+      message: OK
+version: 1