Add base assertions to js sdk

src/testing/api.ts

@@ -339,6 +339,7 @@ export async function sendEvaluation(args: {
         score: args.evaluation.score,
         threshold: args.evaluation.threshold,
         metadata: args.evaluation.metadata,
+        assertions: args.evaluation.assertions,
       },
     });
     return;
@@ -351,6 +352,7 @@ export async function sendEvaluation(args: {
       passed: determineIfEvaluationPassed({ evaluation: args.evaluation }),
       threshold: args.evaluation.threshold,
       metadata: args.evaluation.metadata,
+      assertions: args.evaluation.assertions,
     },
   });
 }

src/testing/evaluators/assertions.ts

@@ -0,0 +1,42 @@
+import { type Assertion, BaseTestEvaluator } from '../models';
+
+/**
+ * Base evaluator for creating an assertions evaluator.
+ */
+export abstract class BaseAssertions<
+  TestCaseType,
+  OutputType,
+> extends BaseTestEvaluator<TestCaseType, OutputType> {
+  /**
+   * Implement your assertion logic and return an array of assertions.
+   * Return undefined if you want to skip evaluation for this test case.
+   */
+  abstract evaluateAssertions(args: {
+    testCase: TestCaseType;
+    output: OutputType;
+  }): Assertion[] | undefined | Promise<Assertion[] | undefined>;
+
+  async evaluateTestCase(args: { testCase: TestCaseType; output: OutputType }) {
+    const assertionsResult = await this.evaluateAssertions({
+      testCase: args.testCase,
+      output: args.output,
+    });
+
+    if (assertionsResult === undefined || assertionsResult.length === 0) {
+      return undefined;
+    }
+
+    // Passes if all required assertions pass
+    const passed = assertionsResult
+      .filter((assertion) => assertion.required)
+      .every((assertion) => assertion.passed);
+
+    return {
+      score: passed ? 1 : 0,
+      threshold: {
+        gte: 1,
+      },
+      assertions: assertionsResult,
+    };
+  }
+}

src/testing/index.ts

@@ -3,6 +3,7 @@ export {
   BaseTestEvaluator,
   BaseEventEvaluator,
   BaseEvaluator,
+  type Assertion,
   type Threshold,
   type Evaluation,
   type TracerEvent,
@@ -30,3 +31,4 @@ export { BaseLLMJudge } from './evaluators/llm-judge';
 export { BaseNSFW } from './evaluators/nsfw';
 export { BaseToxicity } from './evaluators/toxicity';
 export { BaseAccuracy } from './evaluators/accuracy';
+export { BaseAssertions } from './evaluators/assertions';

src/testing/models.ts

@@ -7,10 +7,18 @@ export interface Threshold {
   gte?: number;
 }
 
+export interface Assertion {
+  criterion: string;
+  passed: boolean;
+  required: boolean;
+  metadata?: ArbitraryProperties;
+}
+
 export interface Evaluation {
   score: number;
   threshold?: Threshold;
   metadata?: ArbitraryProperties;
+  assertions?: Assertion[];
 }
 
 export interface TracerEvent {

test/testing/evaluators-logic.spec.ts

@@ -3,6 +3,8 @@ import {
   BaseHasAllSubstrings,
   BaseIsEquals,
   BaseIsValidJSON,
+  BaseAssertions,
+  Assertion,
 } from '../../src/testing';
 import crypto from 'crypto';
 import { isMatch } from 'lodash';
@@ -221,4 +223,99 @@ describe('OOB Evaluators', () => {
       },
     });
   });
+
+  it('BaseAssertions', async () => {
+    interface TestCase {
+      input: string;
+      assertions: {
+        criterion: string;
+        required: boolean;
+      }[];
+    }
+
+    class AssertionsEvaluator extends BaseAssertions<TestCase, string> {
+      id = 'assertion';
+
+      evaluateAssertions(args: {
+        testCase: TestCase;
+        output: string;
+      }): Assertion[] | undefined | Promise<Assertion[] | undefined> {
+        return args.testCase.assertions.map((assertion) => ({
+          criterion: assertion.criterion,
+          passed: args.output.includes(assertion.criterion),
+          required: assertion.required,
+        }));
+      }
+    }
+    await runTestSuite<TestCase, string>({
+      id: 'my-test-id',
+      testCases: [
+        {
+          input: 'hello world',
+          assertions: [
+            {
+              criterion: 'hello',
+              required: true,
+            },
+            {
+              criterion: 'hi',
+              required: false,
+            },
+          ],
+        },
+        {
+          input: 'hi world',
+          assertions: [
+            {
+              criterion: 'hello',
+              required: true,
+            },
+          ],
+        },
+      ],
+      testCaseHash: (testCase) => md5(testCase.input),
+      evaluators: [new AssertionsEvaluator()],
+      fn: ({ testCase }: { testCase: TestCase }) => testCase.input,
+    });
+
+    expectPostRequest({
+      path: '/evals',
+      body: {
+        testExternalId: 'my-test-id',
+        testCaseHash: md5('hello world'),
+        evaluatorExternalId: 'assertion',
+        score: 1,
+        threshold: { gte: 1 },
+        assertions: [
+          {
+            criterion: 'hello',
+            passed: true,
+            required: true,
+          },
+          {
+            criterion: 'hi',
+            passed: false,
+            required: false,
+          },
+        ],
+      },
+    });
+    expectPostRequest({
+      path: '/evals',
+      body: {
+        testExternalId: 'my-test-id',
+        testCaseHash: md5('hi world'),
+        evaluatorExternalId: 'assertion',
+        score: 0,
+        threshold: { gte: 1 },
+        assertions: [
+          {
+            criterion: 'hello',
+            passed: false,
+            required: true,
+          },
+        ],
+      },
+    });
+  });
 });