Add a manifest of all autoevals in typescript and fix export error (#43)

The manifest includes a list of all autoevaluators, that currently must
be kept up-to-date manually.

The export fix clarifies the export as `export type`, because
`index.mjs` from `@braintrust/core` is actually empty.

js/index.ts

@@ -27,9 +27,10 @@
  * @module autoevals
  */
 
-export { Score, ScorerArgs, Scorer } from "@braintrust/core";
+export type { Score, ScorerArgs, Scorer } from "@braintrust/core";
 export * from "./llm";
 export * from "./string";
 export * from "./number";
 export * from "./json";
 export * from "./templates";
+export { Evaluators } from "./manifest";

js/llm.ts

@@ -78,6 +78,8 @@ export async function OpenAIClassifier<RenderArgs, Output>(
     openAiApiKey,
     openAiOrganizationId,
     openAiBaseUrl,
+    openAiDefaultHeaders,
+    openAiDangerouslyAllowBrowser,
     ...remaining
   } = args;
 
@@ -118,7 +120,7 @@ export async function OpenAIClassifier<RenderArgs, Output>(
 
   const messages: ChatCompletionMessageParam[] = messagesArg.map((m) => ({
     ...m,
-    content: m.content && mustache.render(m.content as string, renderArgs),
+    content: m.content ? mustache.render(m.content as string, renderArgs) : "",
   }));
 
   try {
@@ -135,6 +137,8 @@ export async function OpenAIClassifier<RenderArgs, Output>(
         openAiApiKey,
         openAiOrganizationId,
         openAiBaseUrl,
+        openAiDefaultHeaders,
+        openAiDangerouslyAllowBrowser,
       }
     );
 

js/manifest.ts

@@ -0,0 +1,43 @@
+import { Scorer } from "@braintrust/core";
+import { JSONDiff } from "./json";
+import {
+  Battle,
+  ClosedQA,
+  Factuality,
+  Humor,
+  Possible,
+  Security,
+  Sql,
+  Summary,
+  Translation,
+} from "./llm";
+import { NumericDiff } from "./number";
+import { EmbeddingSimilarity, Levenshtein } from "./string";
+
+export const Evaluators: {
+  label: string;
+  methods: Scorer<any, any>[];
+}[] = [
+  {
+    label: "Model-based classification",
+    methods: [
+      Battle,
+      ClosedQA,
+      Humor,
+      Factuality,
+      Possible,
+      Security,
+      Sql,
+      Summary,
+      Translation,
+    ],
+  },
+  {
+    label: "Embeddings",
+    methods: [EmbeddingSimilarity],
+  },
+  {
+    label: "Heuristic",
+    methods: [JSONDiff, Levenshtein, NumericDiff],
+  },
+];

js/oai.ts

@@ -26,17 +26,27 @@ export interface OpenAIAuth {
   openAiApiKey?: string;
   openAiOrganizationId?: string;
   openAiBaseUrl?: string;
+  openAiDefaultHeaders?: Record<string, string>;
+  openAiDangerouslyAllowBrowser?: boolean;
 }
 
 const PROXY_URL = "https://braintrustproxy.com/v1";
 
 export function buildOpenAIClient(options: OpenAIAuth): OpenAI {
-  const { openAiApiKey, openAiOrganizationId, openAiBaseUrl } = options;
+  const {
+    openAiApiKey,
+    openAiOrganizationId,
+    openAiBaseUrl,
+    openAiDefaultHeaders,
+    openAiDangerouslyAllowBrowser,
+  } = options;
 
   return new OpenAI({
     apiKey: openAiApiKey || Env.OPENAI_API_KEY,
     organization: openAiOrganizationId,
     baseURL: openAiBaseUrl || PROXY_URL,
+    defaultHeaders: openAiDefaultHeaders,
+    dangerouslyAllowBrowser: openAiDangerouslyAllowBrowser,
   });
 }
 

package.json

@@ -47,7 +47,7 @@
     "js-levenshtein": "^1.1.6",
     "js-yaml": "^4.1.0",
     "mustache": "^4.2.0",
-    "openai": "^4.12.1",
+    "openai": "^4.23.0",
     "tsx": "^3.12.7"
   }
 }