Support non-strings in valid json

braintrustdata · Dec 15, 2024 · e17b0bb · e17b0bb
1 parent a3ca1cf
commit e17b0bb
Show file tree

Hide file tree

Showing 4 changed files with 50 additions and 6 deletions.
diff --git a/js/json.test.ts b/js/json.test.ts
@@ -100,6 +100,25 @@ test("Valid JSON Test", async () => {
         uniqueItems: true,
       },
     },
+    {
+      output: { a: "1", b: "1" },
+      expected: 1,
+    },
+    {
+      output: [{ a: "1" }, { a: "1", b: 22 }],
+      expected: 1,
+    },
+    {
+      output: 100,
+      expected: 0,
+    },
+    {
+      // This is technically ambiguous, because it _could_ be the valid parsed JSON value
+      // or an unparsed, invalid JSON value. However, since structured outputs _only_ return
+      // JSON values, we can safely assume that any strings are unparsed values.
+      output: "100",
+      expected: 0,
+    },
   ];
 
   for (const { output, expected, schema } of cases) {

diff --git a/js/json.ts b/js/json.ts
@@ -41,14 +41,16 @@ export const JSONDiff: ScorerWithPartial<
  * A binary scorer that evaluates the validity of JSON output, optionally validating against a
  * JSON Schema definition (see https://json-schema.org/learn/getting-started-step-by-step#create).
  */
-export const ValidJSON: ScorerWithPartial<string, { schema?: any }> =
-  makePartial(async ({ output, schema }) => {
+export const ValidJSON: ScorerWithPartial<any, { schema?: any }> = makePartial(
+  async ({ output, schema }) => {
     return {
       name: "ValidJSON",
       score: validJSON(output, schema),
       metadata: { schema },
     };
-  }, "ValidJSON");
+  },
+  "ValidJSON",
+);
 
 async function jsonDiff(
   o1: any,
@@ -151,9 +153,9 @@ const replacer = (key: string, value: any) =>
         }, {})
     : value;
 
-function validJSON<T>(output: string, schema?: Schema | JSONSchemaType<T>) {
+function validJSON<T>(output: any, schema?: Schema | JSONSchemaType<T>) {
   try {
-    const parsed = JSON.parse(output);
+    const parsed = typeof output === "string" ? JSON.parse(output) : output;
 
     if (schema) {
       return validateSchema(parsed, schema);

diff --git a/py/autoevals/json.py b/py/autoevals/json.py
@@ -72,7 +72,7 @@ def _run_eval_sync(self, output, schema=None, **kwargs):
 
     def valid_json(self, output, schema=None):
         try:
-            parsed = json.loads(output)
+            parsed = json.loads(output) if isinstance(output, str) else output
 
             if schema is not None:
                 validate(parsed, schema)

diff --git a/py/autoevals/test_json.py b/py/autoevals/test_json.py
@@ -94,6 +94,29 @@ def test_valid_json():
                 "uniqueItems": True,
             },
         ),
+        (
+            {"a": "1", "b": "1"},
+            1,
+            None,
+        ),
+        (
+            [{"a": "1"}, {"a": "1", "b": 22}],
+            1,
+            None,
+        ),
+        (
+            100,
+            0,
+            None,
+        ),
+        (
+            # This is technically ambiguous, because it _could_ be the valid parsed JSON value
+            # or an unparsed, invalid JSON value. However, since structured outputs _only_ return
+            # JSON values, we can safely assume that any strings are unparsed values.
+            "100",
+            0,
+            None,
+        ),
     ]
 
     evaluator = ValidJSON()