Split evaluation into sub-plugin of vertexai plugin

js/plugins/vertexai/package.json

@@ -66,9 +66,19 @@
       "import": "./lib/index.mjs",
       "types": "./lib/index.d.ts",
       "default": "./lib/index.js"
+    },
+    "./evaluation": {
+      "require": "./lib/evaluation/index.js",
+      "import": "./lib/evaluation/index.mjs",
+      "types": "./lib/evaluation/index.d.ts",
+      "default": "./lib/evaluation/index.js"
     }
   },
   "typesVersions": {
-    "*": {}
+    "*": {
+      "evaluation": [
+        "./lib/evaluation/index"
+      ]
+    }
   }
 }

js/plugins/vertexai/src/evaluator_factory.ts → js/plugins/vertexai/src/evaluation/evaluator_factory.ts

@@ -39,7 +39,7 @@ export class EvaluatorFactory {
   ): Action {
     return defineEvaluator(
       {
-        name: `vertexai/${config.metric.toLocaleLowerCase()}`,
+        name: `vertexaiEvaluation/${config.metric.toLocaleLowerCase()}`,
         displayName: config.displayName,
         definition: config.definition,
       },

js/plugins/vertexai/src/evaluation/index.ts

@@ -0,0 +1,60 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { genkitPlugin, Plugin } from 'genkit';
+import { authenticate } from '../common/auth.js';
+import { confError, DEFAULT_LOCATION } from '../common/global.js';
+import { BasePluginOptions } from '../common/types.js';
+import {
+  VertexAIEvaluationMetric,
+  VertexAIEvaluationMetricType,
+  vertexEvaluators,
+} from './evaluation.js';
+
+export { VertexAIEvaluationMetricType as VertexAIEvaluationMetricType };
+
+export interface PluginOptions extends BasePluginOptions {
+  metrics: VertexAIEvaluationMetric[];
+}
+
+/**
+ *  Plugin for Vertex AI Model Garden
+ */
+export const vertexAIEvaluation: Plugin<[PluginOptions] | []> = genkitPlugin(
+  'vertexaiEvaluation',
+  async (options?: PluginOptions) => {
+    // Authenticate with Google Cloud
+    const authOptions = options?.googleAuth;
+    const authClient = authenticate(authOptions);
+
+    const projectId = options?.projectId || (await authClient.getProjectId());
+    const location = options?.location || DEFAULT_LOCATION;
+
+    if (!location) {
+      throw confError('location', 'GCLOUD_LOCATION');
+    }
+    if (!projectId) {
+      throw confError('project', 'GCLOUD_PROJECT');
+    }
+
+    const metrics =
+      options && options?.metrics.length > 0 ? options?.metrics : [];
+
+    return {
+      evaluators: vertexEvaluators(authClient, metrics, projectId, location),
+    };
+  }
+);
+
+export default vertexAIEvaluation;

js/plugins/vertexai/src/index.ts

@@ -40,11 +40,6 @@ import {
   textEmbeddingGeckoMultilingual001,
   textMultilingualEmbedding002,
 } from './embedder.js';
-import {
-  VertexAIEvaluationMetric,
-  VertexAIEvaluationMetricType,
-  vertexEvaluators,
-} from './evaluation.js';
 import {
   gemini15Flash,
   gemini15FlashPreview,
@@ -114,7 +109,6 @@ export {
   textEmbeddingGecko003,
   textEmbeddingGeckoMultilingual001,
   textMultilingualEmbedding002,
-  VertexAIEvaluationMetricType as VertexAIEvaluationMetricType,
 };
 
 export interface PluginOptions {
@@ -124,10 +118,6 @@ export interface PluginOptions {
   location: string;
   /** Provide custom authentication configuration for connecting to Vertex AI. */
   googleAuth?: GoogleAuthOptions;
-  /** Configure Vertex AI evaluators */
-  evaluation?: {
-    metrics: VertexAIEvaluationMetric[];
-  };
   /**
    * @deprecated use `modelGarden.models`
    */
@@ -176,10 +166,6 @@ export const vertexAI: Plugin<[PluginOptions] | []> = genkitPlugin(
       }
       return vertexClientFactoryCache[requestLocation];
     };
-    const metrics =
-      options?.evaluation && options.evaluation.metrics.length > 0
-        ? options.evaluation.metrics
-        : [];
 
     const models = [
       ...Object.keys(SUPPORTED_IMAGEN_MODELS).map((name) =>
@@ -257,7 +243,6 @@ export const vertexAI: Plugin<[PluginOptions] | []> = genkitPlugin(
     return {
       models,
       embedders,
-      evaluators: vertexEvaluators(authClient, metrics, projectId, location),
       retrievers,
       indexers,
       rerankers,

js/testapps/dev-ui-gallery/src/index.ts

@@ -14,6 +14,127 @@
  * limitations under the License.
  */
 
+import { devLocalVectorstore } from '@genkit-ai/dev-local-vectorstore';
+import { genkitEval, GenkitMetric } from '@genkit-ai/evaluator';
+import { geminiPro, googleAI } from '@genkit-ai/googleai';
+import {
+  claude3Haiku,
+  claude3Opus,
+  claude3Sonnet,
+  textEmbeddingGecko,
+  vertexAI,
+} from '@genkit-ai/vertexai';
+import {
+  vertexAIEvaluation,
+  VertexAIEvaluationMetricType,
+} from '@genkit-ai/vertexai/evaluation';
+import { dotprompt, genkit } from 'genkit';
+import { chroma } from 'genkitx-chromadb';
+import { ollama } from 'genkitx-ollama';
+import { pinecone } from 'genkitx-pinecone';
+
+// Turn off safety checks for evaluation so that the LLM as an evaluator can
+// respond appropriately to potentially harmful content without error.
+export const PERMISSIVE_SAFETY_SETTINGS: any = {
+  safetySettings: [
+    {
+      category: 'HARM_CATEGORY_HATE_SPEECH',
+      threshold: 'BLOCK_NONE',
+    },
+    {
+      category: 'HARM_CATEGORY_DANGEROUS_CONTENT',
+      threshold: 'BLOCK_NONE',
+    },
+    {
+      category: 'HARM_CATEGORY_HARASSMENT',
+      threshold: 'BLOCK_NONE',
+    },
+    {
+      category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT',
+      threshold: 'BLOCK_NONE',
+    },
+  ],
+};
+
+export const ai = genkit({
+  // load at least one plugin representing each action type
+  plugins: [
+    // model providers
+    googleAI({
+      apiVersion: ['v1', 'v1beta'],
+    }),
+    ollama({
+      models: [
+        { name: 'llama2' },
+        { name: 'llama3' },
+        {
+          name: 'gemma',
+          type: 'generate',
+        },
+      ],
+      serverAddress: 'http://127.0.0.1:11434', // default local address
+    }),
+    vertexAI({
+      location: 'us-central1',
+      modelGardenModels: [claude3Haiku, claude3Sonnet, claude3Opus],
+    }),
+    vertexAIEvaluation({
+      location: 'us-central1',
+      metrics: [
+        VertexAIEvaluationMetricType.BLEU,
+        VertexAIEvaluationMetricType.GROUNDEDNESS,
+        VertexAIEvaluationMetricType.SAFETY,
+        {
+          type: VertexAIEvaluationMetricType.ROUGE,
+          metricSpec: {
+            rougeType: 'rougeLsum',
+            useStemmer: true,
+            splitSummaries: 'true',
+          },
+        },
+      ],
+    }),
+
+    // vector stores
+    chroma([
+      {
+        collectionName: 'chroma-collection',
+        embedder: textEmbeddingGecko,
+        embedderOptions: { taskType: 'RETRIEVAL_DOCUMENT' },
+      },
+    ]),
+    devLocalVectorstore([
+      {
+        indexName: 'naive-index',
+        embedder: textEmbeddingGecko,
+        embedderOptions: { taskType: 'RETRIEVAL_DOCUMENT' },
+      },
+    ]),
+    pinecone([
+      {
+        indexId: 'pinecone-index',
+        embedder: textEmbeddingGecko,
+        embedderOptions: { taskType: 'RETRIEVAL_DOCUMENT' },
+      },
+    ]),
+
+    // evaluation
+    genkitEval({
+      judge: geminiPro,
+      judgeConfig: PERMISSIVE_SAFETY_SETTINGS,
+      embedder: textEmbeddingGecko,
+      metrics: [
+        GenkitMetric.ANSWER_RELEVANCY,
+        GenkitMetric.FAITHFULNESS,
+        GenkitMetric.MALICIOUSNESS,
+      ],
+    }),
+
+    // prompt files
+    dotprompt({ dir: './prompts' }),
+  ],
+});
+
 export * from './main/flows-firebase-functions.js';
 export * from './main/flows.js';
 export * from './main/prompts.js';