add openai api key

.github/workflows/ci.yml

@@ -36,6 +36,8 @@ jobs:
           run: poetry install
         - name: pytest
           run: poetry run pytest -v
+          env:
+            OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
 
     python_lint:
         runs-on: ubuntu-latest

dewy/common/collection_embeddings.py

@@ -214,6 +214,10 @@ async def ingest(self, document_id: int, url: str) -> None:
             async with conn.transaction():
 
                 def encode_chunk(c: str) -> str:
+                    # We believe that either invalid unicode or the occurrence of nulls was
+                    # causing problems that *looked* like only the first page from a PDF was
+                    # being indexed (https://github.com/DewyKB/dewy/issues/20). We do not
+                    # know that all of this is truly necessary.
                     encoded = c.encode("utf-8").decode("utf-8", "ignore")
                     return encoded.replace("\x00", "\uFFFD")
 

tests/test_e2e.py

@@ -22,7 +22,7 @@ async def create_collection(client, text_embedding_model: str) -> int:
 async def ingest(client, collection: int, url: str) -> int:
     add_request = AddDocumentRequest(collection_id=collection, url=url)
     add_response = await client.put(
-        "/api/documents/", data=add_request.model_dump_json()
+        "/api/documents/", content=add_request.model_dump_json()
     )
     assert add_response.status_code == 200
 
@@ -52,7 +52,7 @@ async def retrieve(client, collection: int, query: str) -> RetrieveResponse:
         collection_id=collection, query=query, include_image_chunks=False
     )
 
-    response = await client.post("/api/chunks/retrieve", data=request.model_dump_json())
+    response = await client.post("/api/chunks/retrieve", content=request.model_dump_json())
     assert response.status_code == 200
     return RetrieveResponse.model_validate_json(response.content)