test: adding test for 500 sink

when sink fails it should cache report but proceed as usual

server/server/api.py

@@ -105,6 +105,11 @@ async def presign_report_url(
     )
 
 
+@app.post("/500")
+async def error_500():
+    raise HTTPException(500)
+
+
 @app.post("/report", status_code=200)
 @app.put("/report", status_code=200)
 async def accept_report(

tests/functional/chalk/runner.py

@@ -108,7 +108,8 @@ def datetime(self):
 
     @classmethod
     def from_json(cls, data: str):
-        return cls(json.loads(data)[0])
+        info = json.loads(data)
+        return cls(info if isinstance(info, dict) else info[0])
 
 
 class ChalkMark(ContainsMixin, dict):
@@ -254,6 +255,23 @@ def vmark(self):
         assert len(marks) == 1
         return marks[0]
 
+    @property
+    def logged_reports_path(self):
+        return Path.cwd() / "chalk-reports.jsonl"
+
+    @property
+    def logged_reports(self):
+        return [
+            ChalkReport.from_json(json.loads(i)["$message"])
+            for i in self.logged_reports_path.read_text().splitlines()
+        ]
+
+    @property
+    def logged_report(self):
+        reports = self.logged_reports
+        assert len(reports) == 1
+        return reports[0]
+
 
 class Chalk:
     def __init__(
@@ -382,6 +400,7 @@ def insert(
         ignore_errors: bool = False,
         expecting_report: bool = True,
         expecting_chalkmarks: bool = True,
+        use_embedded: bool = True,
     ) -> ChalkProgram:
         result = self.run(
             command="insert",
@@ -392,6 +411,7 @@ def insert(
             env=env,
             ignore_errors=ignore_errors,
             expecting_report=expecting_report,
+            use_embedded=use_embedded,
         )
         if expecting_report:
             if expecting_chalkmarks:

tests/functional/test_sink.py

@@ -174,7 +174,7 @@ def test_post_http_fastapi(
     _test_server(
         artifact=copy_files[0],
         chalk=chalk,
-        conf="post_https_local.c4m",
+        conf="post_http_local.c4m",
         url=server_http,
         server_sql=server_sql,
         verify=None,
@@ -212,6 +212,31 @@ def test_post_https_fastapi(
     )
 
 
+@pytest.mark.parametrize("copy_files", [[CAT_PATH]], indirect=True)
+def test_500_http_fastapi(
+    copy_files: list[Path],
+    chalk: Chalk,
+    server_sql: Callable[[str], str | None],
+    server_http: str,
+):
+    build = _test_server(
+        artifact=copy_files[0],
+        chalk=chalk,
+        conf="post_http_local.c4m",
+        url=server_http,
+        server_sql=server_sql,
+        verify=None,
+        env={
+            "CHALK_POST_URL": f"{SERVER_HTTP}/500",
+            # testing if chalk at least parses headers correctly
+            "CHALK_POST_HEADERS": "x-test-header: test-header",
+        },
+        ignore_errors=True,
+    )
+    assert "publish failures will be cached" in build.logs
+    assert build.logged_report.contains(build.report)
+
+
 @pytest.mark.parametrize("copy_files", [[CAT_PATH]], indirect=True)
 def test_presign_http_fastapi(
     copy_files: list[Path],
@@ -242,24 +267,28 @@ def _test_server(
     server_sql: Callable[[str], str | None],
     verify: str | None,
     env: dict[str, str],
+    ignore_errors=False,
 ):
     initial_chalks_count = int(server_sql("SELECT count(*) FROM chalks") or 0)
 
     # post url must be set
     assert env["CHALK_POST_URL"] != "", "post url is not set"
 
     config = SINK_CONFIGS / conf
-    proc = chalk.run(
-        command="insert",
-        target=artifact,
+    proc = chalk.insert(
+        artifact,
         config=config,
         use_embedded=False,
         env=env,
+        ignore_errors=ignore_errors,
     )
 
     metadata_id = proc.mark["METADATA_ID"]
     assert metadata_id, "metadata id for created chalk not found in stderr"
 
+    if ignore_errors:
+        return proc
+
     db_id = server_sql(f"SELECT chalk_id FROM chalks WHERE metadata_id='{metadata_id}'")
     assert db_id is not None
 
@@ -274,3 +303,5 @@ def _test_server(
     response.raise_for_status()
     fetched_chalk = response.json()
     assert fetched_chalk["METADATA_ID"] == metadata_id
+
+    return proc