From eb5a411fcf3450da24498e91bb7434141019257c Mon Sep 17 00:00:00 2001 From: Kuhr Date: Mon, 19 Aug 2024 00:47:28 +0200 Subject: [PATCH 01/24] Integrate changes from main --- packages/core/src/context.ts | 5 +- packages/core/src/http-client.ts | 8 +- packages/gen-ai-hub/package.json | 1 + .../src/client/openai/openai-client.test.ts | 47 ++++-------- .../src/client/openai/openai-client.ts | 51 ++++++++----- .../src/client/openai/openai-types.ts | 27 ++++++- packages/gen-ai-hub/src/core.test.ts | 74 +++++++++++++++++++ packages/gen-ai-hub/src/core.ts | 47 ++++++++++++ packages/gen-ai-hub/src/index.ts | 10 +-- .../orchestration-client.test.ts | 61 ++++++--------- .../src/orchestration/orchestration-client.ts | 25 +++---- .../src/orchestration/orchestration-types.ts | 11 --- pnpm-lock.yaml | 3 + sample-code/src/aiservice.ts | 33 +++------ test-util/mock-http.ts | 11 ++- tests/e2e-tests/src/orchestration.test.ts | 9 +-- tests/type-tests/test/context.test-d.ts | 4 +- tests/type-tests/test/http-client.test-d.ts | 2 +- tests/type-tests/test/openai.test-d.ts | 19 ++--- tests/type-tests/test/orchestration.test-d.ts | 30 +------- 20 files changed, 272 insertions(+), 206 deletions(-) create mode 100644 packages/gen-ai-hub/src/core.test.ts create mode 100644 packages/gen-ai-hub/src/core.ts diff --git a/packages/core/src/context.ts b/packages/core/src/context.ts index b17b8c1cb..e5e7a37fe 100644 --- a/packages/core/src/context.ts +++ b/packages/core/src/context.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sap-cloud-sdk/util'; import { Destination, + HttpDestination, Service, ServiceCredentials, getServiceBinding, @@ -18,7 +19,7 @@ let aiCoreServiceBinding: Service | undefined; * Returns a destination object from AI Core service binding. * @returns The destination object. */ -export async function getAiCoreDestination(): Promise { +export async function getAiCoreDestination(): Promise { if (!aiCoreServiceBinding) { aiCoreServiceBinding = getAiCoreServiceKeyFromEnv() || getServiceBinding('aicore'); @@ -34,7 +35,7 @@ export async function getAiCoreDestination(): Promise { { useCache: true } - ); + ) as HttpDestination; return aiCoreDestination; } diff --git a/packages/core/src/http-client.ts b/packages/core/src/http-client.ts index a8ba50735..d1d5b17b4 100644 --- a/packages/core/src/http-client.ts +++ b/packages/core/src/http-client.ts @@ -54,19 +54,17 @@ export interface EndpointOptions { * @param requestConfig - The request configuration. * @returns The {@link HttpResponse} from the AI Core service. */ -export async function executeRequest( +export async function executeRequest( endpointOptions: EndpointOptions, - data: Data, + data: any, requestConfig?: CustomRequestConfig ): Promise { const aiCoreDestination = await getAiCoreDestination(); - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const { deploymentConfiguration, ...body } = data; const { url, apiVersion } = endpointOptions; const mergedRequestConfig = { ...mergeWithDefaultRequestConfig(apiVersion, requestConfig), - data: JSON.stringify(body) + data: JSON.stringify(data) }; const targetUrl = aiCoreDestination.url + `/v2/${removeLeadingSlashes(url)}`; diff --git a/packages/gen-ai-hub/package.json b/packages/gen-ai-hub/package.json index c5e8d7806..1e1f02cf5 100644 --- a/packages/gen-ai-hub/package.json +++ b/packages/gen-ai-hub/package.json @@ -28,6 +28,7 @@ }, "dependencies": { "@sap-ai-sdk/core": "workspace:^", + "@sap-ai-sdk/ai-core": "workspace:^", "@sap-cloud-sdk/http-client": "^3.18.0", "@sap-cloud-sdk/connectivity": "^3.18.0", "@sap-cloud-sdk/util": "^3.18.0", diff --git a/packages/gen-ai-hub/src/client/openai/openai-client.test.ts b/packages/gen-ai-hub/src/client/openai/openai-client.test.ts index 35f712a5f..e368a4dd4 100644 --- a/packages/gen-ai-hub/src/client/openai/openai-client.test.ts +++ b/packages/gen-ai-hub/src/client/openai/openai-client.test.ts @@ -1,5 +1,4 @@ import nock from 'nock'; -import { BaseLlmParametersWithDeploymentId } from '@sap-ai-sdk/core'; import { mockClientCredentialsGrantCall, mockInference, @@ -7,23 +6,20 @@ import { } from '../../../../../test-util/mock-http.js'; import { OpenAiChatCompletionOutput, - OpenAiChatCompletionParameters, OpenAiChatMessage, OpenAiEmbeddingOutput, - OpenAiEmbeddingParameters + OpenAiEmbeddingParameters, + OpenAiModels } from './openai-types.js'; import { OpenAiClient } from './openai-client.js'; describe('openai client', () => { - const deploymentConfiguration: BaseLlmParametersWithDeploymentId = { - deploymentId: 'deployment-id' - }; const chatCompletionEndpoint = { - url: `inference/deployments/${deploymentConfiguration.deploymentId}/chat/completions`, + url: `inference/deployments/1234/chat/completions`, apiVersion: '2024-02-01' }; const embeddingsEndpoint = { - url: `inference/deployments/${deploymentConfiguration.deploymentId}/embeddings`, + url: `inference/deployments/1234/embeddings`, apiVersion: '2024-02-01' }; @@ -47,10 +43,7 @@ describe('openai client', () => { } ] as OpenAiChatMessage[] }; - const request: OpenAiChatCompletionParameters = { - ...prompt, - deploymentConfiguration - }; + const mockResponse = parseMockResponse( 'openai', 'openai-chat-completion-success-response.json' @@ -58,7 +51,7 @@ describe('openai client', () => { mockInference( { - data: request + data: prompt }, { data: mockResponse, @@ -67,16 +60,12 @@ describe('openai client', () => { chatCompletionEndpoint ); - const response = await client.chatCompletion(request); + const response = await client.chatCompletion(OpenAiModels.GPT_4o, prompt, '1234'); expect(response).toEqual(mockResponse); }); it('throws on bad request', async () => { const prompt = { messages: [] }; - const request: OpenAiChatCompletionParameters = { - ...prompt, - deploymentConfiguration - }; const mockResponse = parseMockResponse( 'openai', 'openai-error-response.json' @@ -84,7 +73,7 @@ describe('openai client', () => { mockInference( { - data: request + data: prompt }, { data: mockResponse, @@ -93,17 +82,13 @@ describe('openai client', () => { chatCompletionEndpoint ); - expect(client.chatCompletion(request)).rejects.toThrow(); + await expect(client.chatCompletion(OpenAiModels.GPT_4o, prompt, '1234')).rejects.toThrow('status code 400'); }); }); describe('embeddings', () => { it('parses a successful response', async () => { - const prompt = { input: ['AI is fascinating'] }; - const request: OpenAiEmbeddingParameters = { - ...prompt, - deploymentConfiguration - }; + const prompt = { input: ['AI is fascinating'] } as OpenAiEmbeddingParameters; const mockResponse = parseMockResponse( 'openai', 'openai-embeddings-success-response.json' @@ -111,7 +96,7 @@ describe('openai client', () => { mockInference( { - data: request + data: prompt }, { data: mockResponse, @@ -119,16 +104,12 @@ describe('openai client', () => { }, embeddingsEndpoint ); - const response = await client.embeddings(request); + const response = await client.embeddings(OpenAiModels.ADA_002, prompt, '1234'); expect(response).toEqual(mockResponse); }); it('throws on bad request', async () => { const prompt = { input: [] }; - const request: OpenAiEmbeddingParameters = { - ...prompt, - deploymentConfiguration - }; const mockResponse = parseMockResponse( 'openai', 'openai-error-response.json' @@ -136,7 +117,7 @@ describe('openai client', () => { mockInference( { - data: request + data: prompt }, { data: mockResponse, @@ -145,7 +126,7 @@ describe('openai client', () => { embeddingsEndpoint ); - expect(client.embeddings(request)).rejects.toThrow(); + await expect(client.embeddings(OpenAiModels.ADA_002, prompt, '1234')).rejects.toThrow('status code 400'); }); }); }); diff --git a/packages/gen-ai-hub/src/client/openai/openai-client.ts b/packages/gen-ai-hub/src/client/openai/openai-client.ts index 99b0e6a8d..9896d44a4 100644 --- a/packages/gen-ai-hub/src/client/openai/openai-client.ts +++ b/packages/gen-ai-hub/src/client/openai/openai-client.ts @@ -1,14 +1,16 @@ +import { HttpRequestConfig } from '@sap-cloud-sdk/http-client'; +import { CustomRequestConfig, executeRequest } from '@sap-ai-sdk/core'; import { - BaseLlmParameters, - CustomRequestConfig, - executeRequest -} from '@sap-ai-sdk/core'; -import { BaseClient } from '../interface.js'; + DeploymentResolver, + resolveDeployment +} from './../../core.js'; import { OpenAiChatCompletionParameters, OpenAiEmbeddingParameters, OpenAiEmbeddingOutput, - OpenAiChatCompletionOutput + OpenAiChatCompletionOutput, + OpenAiChatModel, + OpenAiEmbeddingModel } from './openai-types.js'; const apiVersion = '2024-02-01'; @@ -16,7 +18,7 @@ const apiVersion = '2024-02-01'; /** * OpenAI GPT Client. */ -export class OpenAiClient implements BaseClient { +export class OpenAiClient { /** * Creates a completion for the chat messages. * @param data - The input parameters for the chat completion. @@ -24,16 +26,16 @@ export class OpenAiClient implements BaseClient { * @returns The completion result. */ async chatCompletion( + model: OpenAiChatModel, data: OpenAiChatCompletionParameters, + deploymentResolver: DeploymentResolver = getDeploymentResolver(model), requestConfig?: CustomRequestConfig ): Promise { + const deployment = typeof deploymentResolver === 'function' ? (await deploymentResolver()).id : deploymentResolver; const response = await executeRequest( - { - url: `/inference/deployments/${data.deploymentConfiguration.deploymentId}/chat/completions`, - apiVersion - }, + { url: `/inference/deployments/${deployment}/chat/completions`, apiVersion: apiVersion }, data, - requestConfig + this.mergeRequestConfig(requestConfig) ); return response.data; } @@ -44,17 +46,32 @@ export class OpenAiClient implements BaseClient { * @returns The completion result. */ async embeddings( + model: OpenAiEmbeddingModel, data: OpenAiEmbeddingParameters, + deploymentResolver: DeploymentResolver = getDeploymentResolver(model), requestConfig?: CustomRequestConfig ): Promise { + const deployment = typeof deploymentResolver === 'function' ? (await deploymentResolver()).id : deploymentResolver; const response = await executeRequest( - { - url: `/inference/deployments/${data.deploymentConfiguration.deploymentId}/embeddings`, - apiVersion - }, + { url: `/inference/deployments/${deployment}/embeddings`, apiVersion: apiVersion }, data, - requestConfig + this.mergeRequestConfig(requestConfig) ); return response.data; } + + mergeRequestConfig(requestConfig?: CustomRequestConfig): HttpRequestConfig { + return { + method: 'POST', + headers: { + 'content-type': 'application/json' + }, + params: { 'api-version': apiVersion }, + ...requestConfig + }; + } } + +function getDeploymentResolver(model: OpenAiChatModel | OpenAiEmbeddingModel) { + return () => resolveDeployment({ scenarioId: 'foundation-models', executableId: 'azure-openai', modelName: model.name, modelVersion: model.version }); +} \ No newline at end of file diff --git a/packages/gen-ai-hub/src/client/openai/openai-types.ts b/packages/gen-ai-hub/src/client/openai/openai-types.ts index 06e04b0cb..cc6c15423 100644 --- a/packages/gen-ai-hub/src/client/openai/openai-types.ts +++ b/packages/gen-ai-hub/src/client/openai/openai-types.ts @@ -1,4 +1,24 @@ -import { BaseLlmParameters } from '@sap-ai-sdk/core'; +export const OpenAiModels = { + // TODO: figure out if these should be constants or functions, e.g. to allow for dynamic versioning + GPT_4o: { name: 'gpt-4o', type: 'chat', version: 'latest' } as OpenAiChatModel, + GPT_4: { name: 'gpt-4', type: 'chat', version: 'latest' } as OpenAiChatModel, + GPT_4_32K: { name: 'gpt-4-32k', type: 'chat', version: 'latest' } as OpenAiChatModel, + GPT_35_TURBO: { name: 'gpt-35-turbo', type: 'chat', version: 'latest' } as OpenAiChatModel, + GPT_35_TURBO_0125: { name: 'gpt-35-turbo-0125', type: 'chat', version: 'latest' } as OpenAiChatModel, + GPT_35_TURBO_16K: { name: 'gpt-35-turbo-16k', type: 'chat', version: 'latest' } as OpenAiChatModel, + ADA_002: { name: 'text-embedding-ada-002', type: 'embedding', version: 'latest' } as OpenAiEmbeddingModel, + TEXT_EMBEDDING_3_SMALL: { name: 'text-embedding-3-small', type: 'embedding', version: 'latest' } as OpenAiEmbeddingModel, + TEXT_EMBEDDING_3_LARGE: { name: 'text-embedding-3-large', type: 'embedding', version: 'latest' } as OpenAiEmbeddingModel, +}; + +export interface OpenAiChatModel { + name: 'gpt-4o' | 'gpt-4' | 'gpt-4-32k' | 'gpt-35-turbo' | 'gpt-35-turbo-0125' | 'gpt-35-turbo-16k'; + version: 'latest'; +}; +export interface OpenAiEmbeddingModel { + name: 'text-embedding-ada-002' | 'text-embedding-3-small' | 'text-embedding-3-large'; + version: 'latest'; +}; /** * OpenAI system message. @@ -254,8 +274,7 @@ export interface OpenAiCompletionParameters { * OpenAI chat completion input parameters. */ export interface OpenAiChatCompletionParameters - extends OpenAiCompletionParameters, - BaseLlmParameters { + extends OpenAiCompletionParameters{ /** * An array of system, user & assistant messages for chat completion. */ @@ -315,7 +334,7 @@ export interface OpenAiChatCompletionParameters /** * OpenAI embedding input parameters. */ -export interface OpenAiEmbeddingParameters extends BaseLlmParameters { +export interface OpenAiEmbeddingParameters { /** * Input text to get embeddings for, encoded as a string. The number of input tokens varies depending on what model you are using. Unless you're embedding code, we suggest replacing newlines (\n) in your input with a single space, as we have observed inferior results when newlines are present. */ diff --git a/packages/gen-ai-hub/src/core.test.ts b/packages/gen-ai-hub/src/core.test.ts new file mode 100644 index 000000000..0e5158df3 --- /dev/null +++ b/packages/gen-ai-hub/src/core.test.ts @@ -0,0 +1,74 @@ +import nock from 'nock'; +import { resolveDeployment } from './core.js'; +import { mockClientCredentialsGrantCall, mockAiCoreEnvVariable, aiCoreDestination } from '../../../test-util/mock-http.js'; + +describe('AICore', () => { + beforeEach(() => { + mockAiCoreEnvVariable(); + mockClientCredentialsGrantCall(); + }); + + afterEach(() => { + nock.cleanAll(); + }); + + it('should be defined', async () => { + nock(aiCoreDestination.url, { + reqheaders: { + 'ai-resource-group': 'default' + }}) + .get( + '/v2/lm/deployments' + ) + .query({ 'scenarioId': 'foundation-models', 'status': 'RUNNING' }) + .reply(200, { + 'count': 1, + 'resources': [ + { + 'configurationId': 'af7a5804-0820-4fbb-8e09-04837b204095', + 'configurationName': 'gpt-4-32k', + 'deploymentUrl': 'https://api.ai.staging.eu-west-1.mlf-aws-dev.com/v2/inference/deployments/d0d49e445e7df086', + 'details': { + 'resources': { + 'backend_details': { + 'model': { + 'name': 'gpt-4-32k', + 'version': 'latest' + } + } + }, + 'scaling': { + 'backend_details': {} + } + }, + 'id': 'd0d49e445e7df086', + 'lastOperation': 'CREATE', + 'status': 'RUNNING' + }] + } + ); + + const result = await resolveDeployment({ scenarioId: 'foundation-models' }); + expect(result).toBeDefined(); + expect(result.id).toBe('d0d49e445e7df086'); + expect(result.configurationName).toBe('gpt-4-32k'); + }); + + it('should throw on empty list', async () => { + nock(aiCoreDestination.url, { + reqheaders: { + 'ai-resource-group': 'default' + }}) + .get( + '/v2/lm/deployments' + ) + .query({ 'scenarioId': 'foundation-models', 'status': 'RUNNING' }) + .reply(200, { + 'count': 0, + 'resources': [] + } + ); + + await expect(resolveDeployment({ scenarioId: 'foundation-models' })).rejects.toThrow('No deployment matched the given criteria'); + }); +}); \ No newline at end of file diff --git a/packages/gen-ai-hub/src/core.ts b/packages/gen-ai-hub/src/core.ts new file mode 100644 index 000000000..a214fcc59 --- /dev/null +++ b/packages/gen-ai-hub/src/core.ts @@ -0,0 +1,47 @@ +import { DeploymentApi, AiDeployment } from '@sap-ai-sdk/ai-core'; + +export type DeploymentResolver = DeploymentId | (() => Promise); +export type DeploymentId = string; + +// TODO: figure out what the best search criteria are +// TODO: discuss to use model?: FoundationModel instead for a search +/** + * Query the AI Core service for a deployment that matches the given criteria. If more than one deployment matches the criteria, the first one is returned. + * @param opts - An object containing the search criteria. A scenario is required, other criteria are optional. + * @param destination - (optional) The destination to use for the request. If not provided, the default AI Core destination is used. + * @returns An AiDeployment, if a deployment was found, fails otherwise. + */ +export async function resolveDeployment(opts: { scenarioId: string; executableId?: string; modelName?: string; modelVersion?: string }): Promise { + // TODO: is there a more elegant way to write this in TS? + let query: any; + if (opts.executableId) { + query = { scenarioId: opts.scenarioId, status: 'RUNNING', executableIds: [opts.executableId] } + } else { + query = { scenarioId: opts.scenarioId, status: 'RUNNING' } + } + + // TODO: add a cache + let deploymentList: AiDeployment[]; + try { + deploymentList = await DeploymentApi + .deploymentQuery(query, { 'AI-Resource-Group': 'default' }) + .execute().then((res) => res.resources); + } catch (error) { + throw new Error('Failed to fetch the list of deployments: ' + error); + } + + if (opts.modelName) { + deploymentList = deploymentList.filter((deployment: any) => modelExtractor(deployment)?.modelName === opts.modelName); + } + if (opts.modelVersion) { + // feature idea: smart handling of 'latest' version + deploymentList = deploymentList.filter((deployment: any) => modelExtractor(deployment)?.modelVersion === opts.modelVersion); + } + if (deploymentList.length === 0) { + // TODO: return undefined instead? + throw new Error('No deployment matched the given criteria: ' + JSON.stringify(opts)); + } + return deploymentList[0]; +} + +const modelExtractor = (deployment: any) => deployment.details?.resources?.backend_details?.model; \ No newline at end of file diff --git a/packages/gen-ai-hub/src/index.ts b/packages/gen-ai-hub/src/index.ts index 73e14135c..7068ce3ab 100644 --- a/packages/gen-ai-hub/src/index.ts +++ b/packages/gen-ai-hub/src/index.ts @@ -3,14 +3,14 @@ export { OpenAiChatCompletionParameters, OpenAiEmbeddingParameters, OpenAiEmbeddingOutput, - OpenAiChatCompletionOutput + OpenAiChatCompletionOutput, + OpenAiModels } from './client/index.js'; export { - GenAiHubClient, - GenAiHubCompletionParameters, - GenAiHubCompletionResponse, + OrchestrationClient, + OrchestrationCompletionParameters, + CompletionPostResponse, PromptConfig, LlmConfig, ChatMessages, - CompletionPostResponse } from './orchestration/index.js'; diff --git a/packages/gen-ai-hub/src/orchestration/orchestration-client.test.ts b/packages/gen-ai-hub/src/orchestration/orchestration-client.test.ts index a1672f977..8d6b8d0b2 100644 --- a/packages/gen-ai-hub/src/orchestration/orchestration-client.test.ts +++ b/packages/gen-ai-hub/src/orchestration/orchestration-client.test.ts @@ -1,23 +1,19 @@ import nock from 'nock'; -import { BaseLlmParametersWithDeploymentId } from '@sap-ai-sdk/core'; import { mockClientCredentialsGrantCall, mockInference, parseMockResponse } from '../../../../test-util/mock-http.js'; import { CompletionPostResponse } from './client/api/index.js'; -import { GenAiHubCompletionParameters } from './orchestration-types.js'; import { - GenAiHubClient, + OrchestrationClient, constructCompletionPostRequest } from './orchestration-client.js'; import { azureContentFilter } from './orchestration-filter-utility.js'; +import { OrchestrationCompletionParameters } from './orchestration-types.js'; describe('GenAiHubClient', () => { - const client = new GenAiHubClient(); - const deploymentConfiguration: BaseLlmParametersWithDeploymentId = { - deploymentId: 'deployment-id' - }; + const client = new OrchestrationClient(); beforeEach(() => { mockClientCredentialsGrantCall(); @@ -28,8 +24,7 @@ describe('GenAiHubClient', () => { }); it('calls chatCompletion with minimum configuration', async () => { - const request: GenAiHubCompletionParameters = { - deploymentConfiguration, + const request = { llmConfig: { model_name: 'gpt-35-turbo-16k', model_params: { max_tokens: 50, temperature: 0.1 } @@ -46,26 +41,22 @@ describe('GenAiHubClient', () => { mockInference( { - data: { - deploymentConfiguration, - ...constructCompletionPostRequest(request) - } + data: constructCompletionPostRequest(request) }, { data: mockResponse, status: 200 }, { - url: `inference/deployments/${deploymentConfiguration.deploymentId}/completion` + url: `inference/deployments/1234/completion` } ); - const response = await client.chatCompletion(request); + const response = await client.chatCompletion(request, '1234'); expect(response).toEqual(mockResponse); }); it('calls chatCompletion with filter configuration supplied using convenience function', async () => { - const request: GenAiHubCompletionParameters = { - deploymentConfiguration, + const request = { llmConfig: { model_name: 'gpt-35-turbo-16k', model_params: { max_tokens: 50, temperature: 0.1 } @@ -88,26 +79,22 @@ describe('GenAiHubClient', () => { mockInference( { - data: { - deploymentConfiguration, - ...constructCompletionPostRequest(request) - } + data: constructCompletionPostRequest(request) }, { data: mockResponse, status: 200 }, { - url: `inference/deployments/${deploymentConfiguration.deploymentId}/completion` + url: `inference/deployments/1234/completion` } ); - const response = await client.chatCompletion(request); + const response = await client.chatCompletion(request, '1234'); expect(response).toEqual(mockResponse); }); it('calls chatCompletion with filtering configuration', async () => { - const request: GenAiHubCompletionParameters = { - deploymentConfiguration, + const request = { llmConfig: { model_name: 'gpt-35-turbo-16k', model_params: { max_tokens: 50, temperature: 0.1 } @@ -142,7 +129,7 @@ describe('GenAiHubClient', () => { ] } } - }; + } as OrchestrationCompletionParameters; const mockResponse = parseMockResponse( 'orchestration', 'genaihub-chat-completion-filter-config.json' @@ -150,26 +137,22 @@ describe('GenAiHubClient', () => { mockInference( { - data: { - deploymentConfiguration, - ...constructCompletionPostRequest(request) - } + data: constructCompletionPostRequest(request) }, { data: mockResponse, status: 200 }, { - url: `inference/deployments/${deploymentConfiguration.deploymentId}/completion` + url: `inference/deployments/1234/completion` } ); - const response = await client.chatCompletion(request); + const response = await client.chatCompletion(request, '1234'); expect(response).toEqual(mockResponse); }); it('sends message history together with templating config', async () => { - const request: GenAiHubCompletionParameters = { - deploymentConfiguration, + const request = { llmConfig: { model_name: 'gpt-35-turbo-16k', model_params: { max_tokens: 50, temperature: 0.1 } @@ -200,20 +183,18 @@ describe('GenAiHubClient', () => { ); mockInference( { - data: { - deploymentConfiguration, - ...constructCompletionPostRequest(request) - } + data: + constructCompletionPostRequest(request) }, { data: mockResponse, status: 200 }, { - url: `inference/deployments/${deploymentConfiguration.deploymentId}/completion` + url: `inference/deployments/1234/completion` } ); - const response = await client.chatCompletion(request); + const response = await client.chatCompletion(request, '1234'); expect(response).toEqual(mockResponse); }); }); diff --git a/packages/gen-ai-hub/src/orchestration/orchestration-client.ts b/packages/gen-ai-hub/src/orchestration/orchestration-client.ts index 7b9c431d7..eef41db0f 100644 --- a/packages/gen-ai-hub/src/orchestration/orchestration-client.ts +++ b/packages/gen-ai-hub/src/orchestration/orchestration-client.ts @@ -1,14 +1,14 @@ import { executeRequest, CustomRequestConfig } from '@sap-ai-sdk/core'; -import { CompletionPostRequest } from './client/api/schema/index.js'; +import { CompletionPostRequest, CompletionPostResponse, OrchestrationConfig } from './client/api/schema/index.js'; import { - GenAiHubCompletionParameters, - GenAiHubCompletionResponse + OrchestrationCompletionParameters } from './orchestration-types.js'; +import { DeploymentResolver, resolveDeployment } from '../core.js'; /** * Get the orchestration client. */ -export class GenAiHubClient { +export class OrchestrationClient { /** * Creates a completion for the chat messages. * @param data - The input parameters for the chat completion. @@ -16,19 +16,18 @@ export class GenAiHubClient { * @returns The completion result. */ async chatCompletion( - data: GenAiHubCompletionParameters, + data: OrchestrationCompletionParameters, + deploymentResolver: DeploymentResolver = () => resolveDeployment({ scenarioId: 'orchestration' }), requestConfig?: CustomRequestConfig - ): Promise { - const dataWithInputParams = { - deploymentConfiguration: data.deploymentConfiguration, - ...constructCompletionPostRequest(data) - }; + ): Promise { + const body = constructCompletionPostRequest(data); + const deployment = typeof deploymentResolver === 'function' ? (await deploymentResolver()).id : deploymentResolver; const response = await executeRequest( { - url: `/inference/deployments/${data.deploymentConfiguration.deploymentId}/completion` + url: `/inference/deployments/${deployment}/completion` }, - dataWithInputParams, + body, requestConfig ); return response.data; @@ -39,7 +38,7 @@ export class GenAiHubClient { * @internal */ export function constructCompletionPostRequest( - input: GenAiHubCompletionParameters + input: OrchestrationCompletionParameters ): CompletionPostRequest { return { orchestration_config: { diff --git a/packages/gen-ai-hub/src/orchestration/orchestration-types.ts b/packages/gen-ai-hub/src/orchestration/orchestration-types.ts index 74c012c0e..fe0ed9638 100644 --- a/packages/gen-ai-hub/src/orchestration/orchestration-types.ts +++ b/packages/gen-ai-hub/src/orchestration/orchestration-types.ts @@ -1,4 +1,3 @@ -import { BaseLlmParameters } from '@sap-ai-sdk/core'; import { ChatMessages, CompletionPostResponse, @@ -7,16 +6,6 @@ import { LLMModuleConfig } from './client/api/index.js'; -/** - * Input Parameters for GenAI hub chat completion. - */ -export type GenAiHubCompletionParameters = BaseLlmParameters & - OrchestrationCompletionParameters; - -/** - * Response for GenAI hub chat completion. - */ -export type GenAiHubCompletionResponse = CompletionPostResponse; /** * Wrapper object to configure prompt. */ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 26f6a0cb9..a88d77d44 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -94,6 +94,9 @@ importers: packages/gen-ai-hub: dependencies: + '@sap-ai-sdk/ai-core': + specifier: workspace:^ + version: link:../ai-core '@sap-ai-sdk/core': specifier: workspace:^ version: link:../core diff --git a/sample-code/src/aiservice.ts b/sample-code/src/aiservice.ts index 36f0543f8..8b1c90100 100644 --- a/sample-code/src/aiservice.ts +++ b/sample-code/src/aiservice.ts @@ -1,22 +1,16 @@ -import { OpenAiClient } from '@sap-ai-sdk/gen-ai-hub'; +// eslint-disable-next-line import/namespace +import { OpenAiClient, OpenAiModels } from '@sap-ai-sdk/gen-ai-hub'; const openAiClient = new OpenAiClient(); -const deployments: { [model: string]: string } = { - 'gpt-4-32k': 'd577d927380c98ea', - 'gpt-35-turbo': 'd66d1927bf590375', - ada: 'd0084a63ebd7bcd3' -}; - /** * Ask GPT about the capital of France. * @returns The answer from GPT. */ export function chatCompletion(): Promise { - const config = getConfig('gpt-35-turbo'); - return openAiClient - .chatCompletion({ - ...config, + return openAiClient.chatCompletion( + OpenAiModels.GPT_35_TURBO, + { messages: [{ role: 'user', content: 'What is the capital of France?' }] }) .then(response => response.choices[0].message.content); @@ -27,19 +21,10 @@ export function chatCompletion(): Promise { * @returns An embedding vector. */ export function computeEmbedding(): Promise { - const config = getConfig('ada'); - return openAiClient - .embeddings({ - ...config, + return openAiClient.embeddings( + OpenAiModels.ADA_002, + { input: 'Hello, world!' }) .then(response => response.data[0].embedding); -} - -function getConfig(model: string) { - return { - deploymentConfiguration: { - deploymentId: deployments[model] - } - }; -} +} \ No newline at end of file diff --git a/test-util/mock-http.ts b/test-util/mock-http.ts index 15f584920..3d6460470 100644 --- a/test-util/mock-http.ts +++ b/test-util/mock-http.ts @@ -89,9 +89,9 @@ export function mockClientCredentialsGrantCall( .reply(responseCode, response); } -export function mockInference( +export function mockInference( request: { - data: D; + data: any; requestConfig?: CustomRequestConfig; }, response: { @@ -100,7 +100,6 @@ export function mockInference( }, endpoint: EndpointOptions = mockEndpoint ): nock.Scope { - const { deploymentConfiguration, ...body } = request.data; const { url, apiVersion } = endpoint; const destination = getMockedAiCoreDestination(); return nock(destination.url, { @@ -108,9 +107,9 @@ export function mockInference( 'ai-resource-group': 'default', authorization: `Bearer ${destination.authTokens?.[0].value}` } - }).post( - `/v2/${url}`, - body as any + }) + .post(`/v2/${url}`, + request.data ) .query(apiVersion ? { 'api-version': apiVersion } : {}) .reply(response.status, response.data); diff --git a/tests/e2e-tests/src/orchestration.test.ts b/tests/e2e-tests/src/orchestration.test.ts index 147e4c54a..4ce24100b 100644 --- a/tests/e2e-tests/src/orchestration.test.ts +++ b/tests/e2e-tests/src/orchestration.test.ts @@ -2,8 +2,8 @@ import path from 'path'; import { fileURLToPath } from 'url'; import dotenv from 'dotenv'; import { - GenAiHubClient, - GenAiHubCompletionParameters + OrchestrationClient, + OrchestrationCompletionParameters } from '@sap-ai-sdk/gen-ai-hub'; // Pick .env file from root directory @@ -13,8 +13,7 @@ dotenv.config({ path: path.resolve(__dirname, '../.env') }); describe('orchestration', () => { it('should complete a chat', async () => { - const request: GenAiHubCompletionParameters = { - deploymentConfiguration: { deploymentId: 'db1d64d9f06be467' }, + const request: OrchestrationCompletionParameters = { llmConfig: { model_name: 'gpt-35-turbo-16k', model_params: { max_tokens: 50, temperature: 0.1 } @@ -23,7 +22,7 @@ describe('orchestration', () => { template: [{ role: 'user', content: 'Hello!' }] } }; - const response = await new GenAiHubClient().chatCompletion(request); + const response = await new OrchestrationClient().chatCompletion(request); expect(response.module_results).toBeDefined(); expect(response.orchestration_result.choices).not.toHaveLength(0); diff --git a/tests/type-tests/test/context.test-d.ts b/tests/type-tests/test/context.test-d.ts index aa967764e..431bf85c1 100644 --- a/tests/type-tests/test/context.test-d.ts +++ b/tests/type-tests/test/context.test-d.ts @@ -1,5 +1,5 @@ -import { Destination } from '@sap-cloud-sdk/connectivity'; +import { HttpDestination } from '@sap-cloud-sdk/connectivity'; import { expectType } from 'tsd'; import { getAiCoreDestination } from '@sap-ai-sdk/core'; -expectType>(getAiCoreDestination()); +expectType>(getAiCoreDestination()); diff --git a/tests/type-tests/test/http-client.test-d.ts b/tests/type-tests/test/http-client.test-d.ts index 918a9f65f..d5287a79f 100644 --- a/tests/type-tests/test/http-client.test-d.ts +++ b/tests/type-tests/test/http-client.test-d.ts @@ -20,6 +20,6 @@ expectError( expectError( executeRequest( {}, - { deploymentConfiguration: { deploymentId: 'id' }, prompt: 'test prompt' } + { prompt: 'test prompt' } ) ); diff --git a/tests/type-tests/test/openai.test-d.ts b/tests/type-tests/test/openai.test-d.ts index 443662e65..d4b0d66e0 100644 --- a/tests/type-tests/test/openai.test-d.ts +++ b/tests/type-tests/test/openai.test-d.ts @@ -2,7 +2,8 @@ import { expectError, expectType } from 'tsd'; import { OpenAiClient, OpenAiChatCompletionOutput, - OpenAiEmbeddingOutput + OpenAiEmbeddingOutput, + OpenAiModels } from '@sap-ai-sdk/gen-ai-hub'; const client = new OpenAiClient(); @@ -12,14 +13,9 @@ expectType(client); * Chat Completion. */ expectType>( - client.chatCompletion({ - deploymentConfiguration: { deploymentId: 'id' }, - messages: [{ role: 'user', content: 'test prompt' }] - }) -); - -expectError( - client.chatCompletion({ + client.chatCompletion( + OpenAiModels.GPT_35_TURBO, + { messages: [{ role: 'user', content: 'test prompt' }] }) ); @@ -28,10 +24,9 @@ expectError( * Embeddings. */ expectType>( - client.embeddings({ - deploymentConfiguration: { deploymentId: 'id' }, + client.embeddings(OpenAiModels.ADA_002,{ input: 'test input' }) ); -expectError(client.embeddings({ input: 'test input' })); +expectError(client.embeddings(OpenAiModels.GPT_35_TURBO,{ input: 'test input' })); diff --git a/tests/type-tests/test/orchestration.test-d.ts b/tests/type-tests/test/orchestration.test-d.ts index 82cb0dd16..7a5e38df1 100644 --- a/tests/type-tests/test/orchestration.test-d.ts +++ b/tests/type-tests/test/orchestration.test-d.ts @@ -1,15 +1,14 @@ import { expectError, expectType } from 'tsd'; -import { GenAiHubClient, CompletionPostResponse } from '@sap-ai-sdk/gen-ai-hub'; +import { OrchestrationClient, CompletionPostResponse } from '@sap-ai-sdk/gen-ai-hub'; -const client = new GenAiHubClient(); -expectType(client); +const client = new OrchestrationClient(); +expectType(client); /** * Chat Completion. */ expectType>( client.chatCompletion({ - deploymentConfiguration: { deploymentId: 'id' }, prompt: { template: [{ role: 'user', content: 'Hello!' }] }, @@ -25,7 +24,6 @@ expectType>( */ expectType>( client.chatCompletion({ - deploymentConfiguration: { deploymentId: 'id' }, prompt: { template: [{ role: 'user', content: 'Hello!' }], messages_history: [ @@ -47,28 +45,11 @@ expectType>( }) ); -/** - * Deployment details are mandatory. - */ -expectError>( - client.chatCompletion({ - prompt: { - template: [{ role: 'user', content: 'Hello!' }] - }, - llmConfig: { - model_name: 'gpt-35-turbo-16k', - model_params: {} - } - }) -); - /** * Orchestration completion parameters cannot be empty. */ expectError( - client.chatCompletion({ - deploymentConfiguration: { deploymentId: 'id' } - }) + client.chatCompletion({}) ); /** @@ -76,7 +57,6 @@ expectError( */ expectError( client.chatCompletion({ - deploymentConfiguration: { deploymentId: 'id' }, llmConfig: { model_name: 'gpt-35-turbo-16k', model_params: {} @@ -89,7 +69,6 @@ expectError( */ expectError( client.chatCompletion({ - deploymentConfiguration: { deploymentId: 'id' }, prompt: { template: [{ role: 'user', content: 'Hello!' }] }, @@ -104,7 +83,6 @@ expectError( */ expectType>( client.chatCompletion({ - deploymentConfiguration: { deploymentId: 'id' }, prompt: { template: [{ role: 'user', content: 'Hello!' }] }, From 4bb8713eba7302ff90b127874e37f6e3fd48f58d Mon Sep 17 00:00:00 2001 From: Kuhr Date: Mon, 19 Aug 2024 00:56:24 +0200 Subject: [PATCH 02/24] Fix linting --- packages/core/src/context.ts | 5 +- .../src/client/openai/openai-client.test.ts | 30 +++++-- .../src/client/openai/openai-client.ts | 38 +++++--- .../src/client/openai/openai-types.ts | 69 ++++++++++++--- packages/gen-ai-hub/src/core.test.ts | 86 ++++++++++--------- packages/gen-ai-hub/src/core.ts | 50 ++++++++--- packages/gen-ai-hub/src/index.ts | 2 +- .../orchestration-client.test.ts | 11 ++- .../src/orchestration/orchestration-client.ts | 18 ++-- .../src/orchestration/orchestration-types.ts | 1 - sample-code/src/aiservice.ts | 13 ++- tests/type-tests/test/http-client.test-d.ts | 7 +- tests/type-tests/test/openai.test-d.ts | 10 +-- tests/type-tests/test/orchestration.test-d.ts | 9 +- 14 files changed, 223 insertions(+), 126 deletions(-) diff --git a/packages/core/src/context.ts b/packages/core/src/context.ts index e5e7a37fe..032bf574f 100644 --- a/packages/core/src/context.ts +++ b/packages/core/src/context.ts @@ -1,6 +1,5 @@ import { createLogger } from '@sap-cloud-sdk/util'; import { - Destination, HttpDestination, Service, ServiceCredentials, @@ -30,12 +29,12 @@ export async function getAiCoreDestination(): Promise { } } - const aiCoreDestination = await transformServiceBindingToDestination( + const aiCoreDestination = (await transformServiceBindingToDestination( aiCoreServiceBinding, { useCache: true } - ) as HttpDestination; + )) as HttpDestination; return aiCoreDestination; } diff --git a/packages/gen-ai-hub/src/client/openai/openai-client.test.ts b/packages/gen-ai-hub/src/client/openai/openai-client.test.ts index e368a4dd4..3d1032c28 100644 --- a/packages/gen-ai-hub/src/client/openai/openai-client.test.ts +++ b/packages/gen-ai-hub/src/client/openai/openai-client.test.ts @@ -15,11 +15,11 @@ import { OpenAiClient } from './openai-client.js'; describe('openai client', () => { const chatCompletionEndpoint = { - url: `inference/deployments/1234/chat/completions`, + url: 'inference/deployments/1234/chat/completions', apiVersion: '2024-02-01' }; const embeddingsEndpoint = { - url: `inference/deployments/1234/embeddings`, + url: 'inference/deployments/1234/embeddings', apiVersion: '2024-02-01' }; @@ -43,7 +43,7 @@ describe('openai client', () => { } ] as OpenAiChatMessage[] }; - + const mockResponse = parseMockResponse( 'openai', 'openai-chat-completion-success-response.json' @@ -60,7 +60,11 @@ describe('openai client', () => { chatCompletionEndpoint ); - const response = await client.chatCompletion(OpenAiModels.GPT_4o, prompt, '1234'); + const response = await client.chatCompletion( + OpenAiModels.GPT_4o, + prompt, + '1234' + ); expect(response).toEqual(mockResponse); }); @@ -82,13 +86,17 @@ describe('openai client', () => { chatCompletionEndpoint ); - await expect(client.chatCompletion(OpenAiModels.GPT_4o, prompt, '1234')).rejects.toThrow('status code 400'); + await expect( + client.chatCompletion(OpenAiModels.GPT_4o, prompt, '1234') + ).rejects.toThrow('status code 400'); }); }); describe('embeddings', () => { it('parses a successful response', async () => { - const prompt = { input: ['AI is fascinating'] } as OpenAiEmbeddingParameters; + const prompt = { + input: ['AI is fascinating'] + } as OpenAiEmbeddingParameters; const mockResponse = parseMockResponse( 'openai', 'openai-embeddings-success-response.json' @@ -104,7 +112,11 @@ describe('openai client', () => { }, embeddingsEndpoint ); - const response = await client.embeddings(OpenAiModels.ADA_002, prompt, '1234'); + const response = await client.embeddings( + OpenAiModels.ADA_002, + prompt, + '1234' + ); expect(response).toEqual(mockResponse); }); @@ -126,7 +138,9 @@ describe('openai client', () => { embeddingsEndpoint ); - await expect(client.embeddings(OpenAiModels.ADA_002, prompt, '1234')).rejects.toThrow('status code 400'); + await expect( + client.embeddings(OpenAiModels.ADA_002, prompt, '1234') + ).rejects.toThrow('status code 400'); }); }); }); diff --git a/packages/gen-ai-hub/src/client/openai/openai-client.ts b/packages/gen-ai-hub/src/client/openai/openai-client.ts index 9896d44a4..838ddfb00 100644 --- a/packages/gen-ai-hub/src/client/openai/openai-client.ts +++ b/packages/gen-ai-hub/src/client/openai/openai-client.ts @@ -1,9 +1,6 @@ import { HttpRequestConfig } from '@sap-cloud-sdk/http-client'; import { CustomRequestConfig, executeRequest } from '@sap-ai-sdk/core'; -import { - DeploymentResolver, - resolveDeployment -} from './../../core.js'; +import { DeploymentResolver, resolveDeployment } from '../../core.js'; import { OpenAiChatCompletionParameters, OpenAiEmbeddingParameters, @@ -21,7 +18,9 @@ const apiVersion = '2024-02-01'; export class OpenAiClient { /** * Creates a completion for the chat messages. + * @param model - The model to use for the chat completion. * @param data - The input parameters for the chat completion. + * @param deploymentResolver - A deployment id or a function to retrieve it. * @param requestConfig - The request configuration. * @returns The completion result. */ @@ -31,9 +30,15 @@ export class OpenAiClient { deploymentResolver: DeploymentResolver = getDeploymentResolver(model), requestConfig?: CustomRequestConfig ): Promise { - const deployment = typeof deploymentResolver === 'function' ? (await deploymentResolver()).id : deploymentResolver; + const deployment = + typeof deploymentResolver === 'function' + ? (await deploymentResolver()).id + : deploymentResolver; const response = await executeRequest( - { url: `/inference/deployments/${deployment}/chat/completions`, apiVersion: apiVersion }, + { + url: `/inference/deployments/${deployment}/chat/completions`, + apiVersion + }, data, this.mergeRequestConfig(requestConfig) ); @@ -41,7 +46,9 @@ export class OpenAiClient { } /** * Creates an embedding vector representing the given text. - * @param data - The input parameters for the chat completion. + * @param model - The model to use for the embedding computation. + * @param data - The text to embed. + * @param deploymentResolver - A deployment id or a function to retrieve it. * @param requestConfig - The request configuration. * @returns The completion result. */ @@ -51,9 +58,12 @@ export class OpenAiClient { deploymentResolver: DeploymentResolver = getDeploymentResolver(model), requestConfig?: CustomRequestConfig ): Promise { - const deployment = typeof deploymentResolver === 'function' ? (await deploymentResolver()).id : deploymentResolver; + const deployment = + typeof deploymentResolver === 'function' + ? (await deploymentResolver()).id + : deploymentResolver; const response = await executeRequest( - { url: `/inference/deployments/${deployment}/embeddings`, apiVersion: apiVersion }, + { url: `/inference/deployments/${deployment}/embeddings`, apiVersion }, data, this.mergeRequestConfig(requestConfig) ); @@ -73,5 +83,11 @@ export class OpenAiClient { } function getDeploymentResolver(model: OpenAiChatModel | OpenAiEmbeddingModel) { - return () => resolveDeployment({ scenarioId: 'foundation-models', executableId: 'azure-openai', modelName: model.name, modelVersion: model.version }); -} \ No newline at end of file + return () => + resolveDeployment({ + scenarioId: 'foundation-models', + executableId: 'azure-openai', + modelName: model.name, + modelVersion: model.version + }); +} diff --git a/packages/gen-ai-hub/src/client/openai/openai-types.ts b/packages/gen-ai-hub/src/client/openai/openai-types.ts index cc6c15423..1b0b8e734 100644 --- a/packages/gen-ai-hub/src/client/openai/openai-types.ts +++ b/packages/gen-ai-hub/src/client/openai/openai-types.ts @@ -1,24 +1,67 @@ +// TODO: docs +/* eslint-disable */ export const OpenAiModels = { // TODO: figure out if these should be constants or functions, e.g. to allow for dynamic versioning - GPT_4o: { name: 'gpt-4o', type: 'chat', version: 'latest' } as OpenAiChatModel, + GPT_4o: { + name: 'gpt-4o', + type: 'chat', + version: 'latest' + } as OpenAiChatModel, GPT_4: { name: 'gpt-4', type: 'chat', version: 'latest' } as OpenAiChatModel, - GPT_4_32K: { name: 'gpt-4-32k', type: 'chat', version: 'latest' } as OpenAiChatModel, - GPT_35_TURBO: { name: 'gpt-35-turbo', type: 'chat', version: 'latest' } as OpenAiChatModel, - GPT_35_TURBO_0125: { name: 'gpt-35-turbo-0125', type: 'chat', version: 'latest' } as OpenAiChatModel, - GPT_35_TURBO_16K: { name: 'gpt-35-turbo-16k', type: 'chat', version: 'latest' } as OpenAiChatModel, - ADA_002: { name: 'text-embedding-ada-002', type: 'embedding', version: 'latest' } as OpenAiEmbeddingModel, - TEXT_EMBEDDING_3_SMALL: { name: 'text-embedding-3-small', type: 'embedding', version: 'latest' } as OpenAiEmbeddingModel, - TEXT_EMBEDDING_3_LARGE: { name: 'text-embedding-3-large', type: 'embedding', version: 'latest' } as OpenAiEmbeddingModel, + GPT_4_32K: { + name: 'gpt-4-32k', + type: 'chat', + version: 'latest' + } as OpenAiChatModel, + GPT_35_TURBO: { + name: 'gpt-35-turbo', + type: 'chat', + version: 'latest' + } as OpenAiChatModel, + GPT_35_TURBO_0125: { + name: 'gpt-35-turbo-0125', + type: 'chat', + version: 'latest' + } as OpenAiChatModel, + GPT_35_TURBO_16K: { + name: 'gpt-35-turbo-16k', + type: 'chat', + version: 'latest' + } as OpenAiChatModel, + ADA_002: { + name: 'text-embedding-ada-002', + type: 'embedding', + version: 'latest' + } as OpenAiEmbeddingModel, + TEXT_EMBEDDING_3_SMALL: { + name: 'text-embedding-3-small', + type: 'embedding', + version: 'latest' + } as OpenAiEmbeddingModel, + TEXT_EMBEDDING_3_LARGE: { + name: 'text-embedding-3-large', + type: 'embedding', + version: 'latest' + } as OpenAiEmbeddingModel }; export interface OpenAiChatModel { - name: 'gpt-4o' | 'gpt-4' | 'gpt-4-32k' | 'gpt-35-turbo' | 'gpt-35-turbo-0125' | 'gpt-35-turbo-16k'; + name: + | 'gpt-4o' + | 'gpt-4' + | 'gpt-4-32k' + | 'gpt-35-turbo' + | 'gpt-35-turbo-0125' + | 'gpt-35-turbo-16k'; version: 'latest'; -}; +} export interface OpenAiEmbeddingModel { - name: 'text-embedding-ada-002' | 'text-embedding-3-small' | 'text-embedding-3-large'; + name: + | 'text-embedding-ada-002' + | 'text-embedding-3-small' + | 'text-embedding-3-large'; version: 'latest'; -}; +} /** * OpenAI system message. @@ -274,7 +317,7 @@ export interface OpenAiCompletionParameters { * OpenAI chat completion input parameters. */ export interface OpenAiChatCompletionParameters - extends OpenAiCompletionParameters{ + extends OpenAiCompletionParameters { /** * An array of system, user & assistant messages for chat completion. */ diff --git a/packages/gen-ai-hub/src/core.test.ts b/packages/gen-ai-hub/src/core.test.ts index 0e5158df3..f2eb69ccd 100644 --- a/packages/gen-ai-hub/src/core.test.ts +++ b/packages/gen-ai-hub/src/core.test.ts @@ -1,6 +1,10 @@ import nock from 'nock'; +import { + mockClientCredentialsGrantCall, + mockAiCoreEnvVariable, + aiCoreDestination +} from '../../../test-util/mock-http.js'; import { resolveDeployment } from './core.js'; -import { mockClientCredentialsGrantCall, mockAiCoreEnvVariable, aiCoreDestination } from '../../../test-util/mock-http.js'; describe('AICore', () => { beforeEach(() => { @@ -11,64 +15,64 @@ describe('AICore', () => { afterEach(() => { nock.cleanAll(); }); - + it('should be defined', async () => { nock(aiCoreDestination.url, { reqheaders: { 'ai-resource-group': 'default' - }}) - .get( - '/v2/lm/deployments' - ) - .query({ 'scenarioId': 'foundation-models', 'status': 'RUNNING' }) + } + }) + .get('/v2/lm/deployments') + .query({ scenarioId: 'foundation-models', status: 'RUNNING' }) .reply(200, { - 'count': 1, - 'resources': [ + count: 1, + resources: [ { - 'configurationId': 'af7a5804-0820-4fbb-8e09-04837b204095', - 'configurationName': 'gpt-4-32k', - 'deploymentUrl': 'https://api.ai.staging.eu-west-1.mlf-aws-dev.com/v2/inference/deployments/d0d49e445e7df086', - 'details': { - 'resources': { - 'backend_details': { - 'model': { - 'name': 'gpt-4-32k', - 'version': 'latest' + configurationId: 'af7a5804-0820-4fbb-8e09-04837b204095', + configurationName: 'gpt-4-32k', + deploymentUrl: + 'https://api.ai.staging.eu-west-1.mlf-aws-dev.com/v2/inference/deployments/d0d49e445e7df086', + details: { + resources: { + backend_details: { + model: { + name: 'gpt-4-32k', + version: 'latest' } } }, - 'scaling': { - 'backend_details': {} + scaling: { + backend_details: {} } }, - 'id': 'd0d49e445e7df086', - 'lastOperation': 'CREATE', - 'status': 'RUNNING' - }] - } - ); + id: 'd0d49e445e7df086', + lastOperation: 'CREATE', + status: 'RUNNING' + } + ] + }); - const result = await resolveDeployment({ scenarioId: 'foundation-models' }); - expect(result).toBeDefined(); - expect(result.id).toBe('d0d49e445e7df086'); - expect(result.configurationName).toBe('gpt-4-32k'); + const result = await resolveDeployment({ scenarioId: 'foundation-models' }); + expect(result).toBeDefined(); + expect(result.id).toBe('d0d49e445e7df086'); + expect(result.configurationName).toBe('gpt-4-32k'); }); it('should throw on empty list', async () => { nock(aiCoreDestination.url, { reqheaders: { 'ai-resource-group': 'default' - }}) - .get( - '/v2/lm/deployments' - ) - .query({ 'scenarioId': 'foundation-models', 'status': 'RUNNING' }) - .reply(200, { - 'count': 0, - 'resources': [] } - ); + }) + .get('/v2/lm/deployments') + .query({ scenarioId: 'foundation-models', status: 'RUNNING' }) + .reply(200, { + count: 0, + resources: [] + }); - await expect(resolveDeployment({ scenarioId: 'foundation-models' })).rejects.toThrow('No deployment matched the given criteria'); + await expect( + resolveDeployment({ scenarioId: 'foundation-models' }) + ).rejects.toThrow('No deployment matched the given criteria'); }); -}); \ No newline at end of file +}); diff --git a/packages/gen-ai-hub/src/core.ts b/packages/gen-ai-hub/src/core.ts index a214fcc59..ca2943444 100644 --- a/packages/gen-ai-hub/src/core.ts +++ b/packages/gen-ai-hub/src/core.ts @@ -1,47 +1,71 @@ import { DeploymentApi, AiDeployment } from '@sap-ai-sdk/ai-core'; +// TODO: docs +/* eslint-disable */ + export type DeploymentResolver = DeploymentId | (() => Promise); export type DeploymentId = string; // TODO: figure out what the best search criteria are -// TODO: discuss to use model?: FoundationModel instead for a search /** * Query the AI Core service for a deployment that matches the given criteria. If more than one deployment matches the criteria, the first one is returned. - * @param opts - An object containing the search criteria. A scenario is required, other criteria are optional. - * @param destination - (optional) The destination to use for the request. If not provided, the default AI Core destination is used. + * @param opts.scenarioId - The scenario ID of the deployment. + * @param opts.executableId - The executable of the deployment. + * @param opts.modelName - The name of the model of the deployment. + * @param opts.modelVersion - The version of the model of the deployment. * @returns An AiDeployment, if a deployment was found, fails otherwise. */ -export async function resolveDeployment(opts: { scenarioId: string; executableId?: string; modelName?: string; modelVersion?: string }): Promise { +export async function resolveDeployment(opts: { + scenarioId: string; + executableId?: string; + modelName?: string; + modelVersion?: string; +}): Promise { // TODO: is there a more elegant way to write this in TS? let query: any; if (opts.executableId) { - query = { scenarioId: opts.scenarioId, status: 'RUNNING', executableIds: [opts.executableId] } + query = { + scenarioId: opts.scenarioId, + status: 'RUNNING', + executableIds: [opts.executableId] + }; } else { - query = { scenarioId: opts.scenarioId, status: 'RUNNING' } + query = { scenarioId: opts.scenarioId, status: 'RUNNING' }; } // TODO: add a cache let deploymentList: AiDeployment[]; try { - deploymentList = await DeploymentApi - .deploymentQuery(query, { 'AI-Resource-Group': 'default' }) - .execute().then((res) => res.resources); + deploymentList = await DeploymentApi.deploymentQuery(query, { + 'AI-Resource-Group': 'default' + }) + .execute() + .then(res => res.resources); } catch (error) { throw new Error('Failed to fetch the list of deployments: ' + error); } if (opts.modelName) { - deploymentList = deploymentList.filter((deployment: any) => modelExtractor(deployment)?.modelName === opts.modelName); + deploymentList = deploymentList.filter( + (deployment: any) => + modelExtractor(deployment)?.modelName === opts.modelName + ); } if (opts.modelVersion) { // feature idea: smart handling of 'latest' version - deploymentList = deploymentList.filter((deployment: any) => modelExtractor(deployment)?.modelVersion === opts.modelVersion); + deploymentList = deploymentList.filter( + (deployment: any) => + modelExtractor(deployment)?.modelVersion === opts.modelVersion + ); } if (deploymentList.length === 0) { // TODO: return undefined instead? - throw new Error('No deployment matched the given criteria: ' + JSON.stringify(opts)); + throw new Error( + 'No deployment matched the given criteria: ' + JSON.stringify(opts) + ); } return deploymentList[0]; } -const modelExtractor = (deployment: any) => deployment.details?.resources?.backend_details?.model; \ No newline at end of file +const modelExtractor = (deployment: any) => + deployment.details?.resources?.backend_details?.model; diff --git a/packages/gen-ai-hub/src/index.ts b/packages/gen-ai-hub/src/index.ts index 7068ce3ab..71ff6d688 100644 --- a/packages/gen-ai-hub/src/index.ts +++ b/packages/gen-ai-hub/src/index.ts @@ -12,5 +12,5 @@ export { CompletionPostResponse, PromptConfig, LlmConfig, - ChatMessages, + ChatMessages } from './orchestration/index.js'; diff --git a/packages/gen-ai-hub/src/orchestration/orchestration-client.test.ts b/packages/gen-ai-hub/src/orchestration/orchestration-client.test.ts index 8d6b8d0b2..56b41ee6b 100644 --- a/packages/gen-ai-hub/src/orchestration/orchestration-client.test.ts +++ b/packages/gen-ai-hub/src/orchestration/orchestration-client.test.ts @@ -48,7 +48,7 @@ describe('GenAiHubClient', () => { status: 200 }, { - url: `inference/deployments/1234/completion` + url: 'inference/deployments/1234/completion' } ); const response = await client.chatCompletion(request, '1234'); @@ -86,7 +86,7 @@ describe('GenAiHubClient', () => { status: 200 }, { - url: `inference/deployments/1234/completion` + url: 'inference/deployments/1234/completion' } ); const response = await client.chatCompletion(request, '1234'); @@ -144,7 +144,7 @@ describe('GenAiHubClient', () => { status: 200 }, { - url: `inference/deployments/1234/completion` + url: 'inference/deployments/1234/completion' } ); const response = await client.chatCompletion(request, '1234'); @@ -183,15 +183,14 @@ describe('GenAiHubClient', () => { ); mockInference( { - data: - constructCompletionPostRequest(request) + data: constructCompletionPostRequest(request) }, { data: mockResponse, status: 200 }, { - url: `inference/deployments/1234/completion` + url: 'inference/deployments/1234/completion' } ); const response = await client.chatCompletion(request, '1234'); diff --git a/packages/gen-ai-hub/src/orchestration/orchestration-client.ts b/packages/gen-ai-hub/src/orchestration/orchestration-client.ts index eef41db0f..f3abf46a0 100644 --- a/packages/gen-ai-hub/src/orchestration/orchestration-client.ts +++ b/packages/gen-ai-hub/src/orchestration/orchestration-client.ts @@ -1,9 +1,10 @@ import { executeRequest, CustomRequestConfig } from '@sap-ai-sdk/core'; -import { CompletionPostRequest, CompletionPostResponse, OrchestrationConfig } from './client/api/schema/index.js'; -import { - OrchestrationCompletionParameters -} from './orchestration-types.js'; import { DeploymentResolver, resolveDeployment } from '../core.js'; +import { + CompletionPostRequest, + CompletionPostResponse +} from './client/api/schema/index.js'; +import { OrchestrationCompletionParameters } from './orchestration-types.js'; /** * Get the orchestration client. @@ -12,16 +13,21 @@ export class OrchestrationClient { /** * Creates a completion for the chat messages. * @param data - The input parameters for the chat completion. + * @param deploymentResolver - A deployment id or a function to retrieve it. * @param requestConfig - Request configuration. * @returns The completion result. */ async chatCompletion( data: OrchestrationCompletionParameters, - deploymentResolver: DeploymentResolver = () => resolveDeployment({ scenarioId: 'orchestration' }), + deploymentResolver: DeploymentResolver = () => + resolveDeployment({ scenarioId: 'orchestration' }), requestConfig?: CustomRequestConfig ): Promise { const body = constructCompletionPostRequest(data); - const deployment = typeof deploymentResolver === 'function' ? (await deploymentResolver()).id : deploymentResolver; + const deployment = + typeof deploymentResolver === 'function' + ? (await deploymentResolver()).id + : deploymentResolver; const response = await executeRequest( { diff --git a/packages/gen-ai-hub/src/orchestration/orchestration-types.ts b/packages/gen-ai-hub/src/orchestration/orchestration-types.ts index fe0ed9638..e993be15f 100644 --- a/packages/gen-ai-hub/src/orchestration/orchestration-types.ts +++ b/packages/gen-ai-hub/src/orchestration/orchestration-types.ts @@ -1,6 +1,5 @@ import { ChatMessages, - CompletionPostResponse, FilteringModuleConfig, InputParamsEntry, LLMModuleConfig diff --git a/sample-code/src/aiservice.ts b/sample-code/src/aiservice.ts index 8b1c90100..ba540f1e3 100644 --- a/sample-code/src/aiservice.ts +++ b/sample-code/src/aiservice.ts @@ -1,4 +1,3 @@ -// eslint-disable-next-line import/namespace import { OpenAiClient, OpenAiModels } from '@sap-ai-sdk/gen-ai-hub'; const openAiClient = new OpenAiClient(); @@ -8,9 +7,8 @@ const openAiClient = new OpenAiClient(); * @returns The answer from GPT. */ export function chatCompletion(): Promise { - return openAiClient.chatCompletion( - OpenAiModels.GPT_35_TURBO, - { + return openAiClient + .chatCompletion(OpenAiModels.GPT_35_TURBO, { messages: [{ role: 'user', content: 'What is the capital of France?' }] }) .then(response => response.choices[0].message.content); @@ -21,10 +19,9 @@ export function chatCompletion(): Promise { * @returns An embedding vector. */ export function computeEmbedding(): Promise { - return openAiClient.embeddings( - OpenAiModels.ADA_002, - { + return openAiClient + .embeddings(OpenAiModels.ADA_002, { input: 'Hello, world!' }) .then(response => response.data[0].embedding); -} \ No newline at end of file +} diff --git a/tests/type-tests/test/http-client.test-d.ts b/tests/type-tests/test/http-client.test-d.ts index d5287a79f..e8cb8b490 100644 --- a/tests/type-tests/test/http-client.test-d.ts +++ b/tests/type-tests/test/http-client.test-d.ts @@ -17,9 +17,4 @@ expectError( ) ); -expectError( - executeRequest( - {}, - { prompt: 'test prompt' } - ) -); +expectError(executeRequest({}, { prompt: 'test prompt' })); diff --git a/tests/type-tests/test/openai.test-d.ts b/tests/type-tests/test/openai.test-d.ts index d4b0d66e0..f07528698 100644 --- a/tests/type-tests/test/openai.test-d.ts +++ b/tests/type-tests/test/openai.test-d.ts @@ -13,9 +13,7 @@ expectType(client); * Chat Completion. */ expectType>( - client.chatCompletion( - OpenAiModels.GPT_35_TURBO, - { + client.chatCompletion(OpenAiModels.GPT_35_TURBO, { messages: [{ role: 'user', content: 'test prompt' }] }) ); @@ -24,9 +22,11 @@ expectType>( * Embeddings. */ expectType>( - client.embeddings(OpenAiModels.ADA_002,{ + client.embeddings(OpenAiModels.ADA_002, { input: 'test input' }) ); -expectError(client.embeddings(OpenAiModels.GPT_35_TURBO,{ input: 'test input' })); +expectError( + client.embeddings(OpenAiModels.GPT_35_TURBO, { input: 'test input' }) +); diff --git a/tests/type-tests/test/orchestration.test-d.ts b/tests/type-tests/test/orchestration.test-d.ts index 7a5e38df1..c85042848 100644 --- a/tests/type-tests/test/orchestration.test-d.ts +++ b/tests/type-tests/test/orchestration.test-d.ts @@ -1,5 +1,8 @@ import { expectError, expectType } from 'tsd'; -import { OrchestrationClient, CompletionPostResponse } from '@sap-ai-sdk/gen-ai-hub'; +import { + OrchestrationClient, + CompletionPostResponse +} from '@sap-ai-sdk/gen-ai-hub'; const client = new OrchestrationClient(); expectType(client); @@ -48,9 +51,7 @@ expectType>( /** * Orchestration completion parameters cannot be empty. */ -expectError( - client.chatCompletion({}) -); +expectError(client.chatCompletion({})); /** * Prompt templates cannot be empty. From c39e009b993b0ae316bba3e20795acb1d549644a Mon Sep 17 00:00:00 2001 From: Kuhr Date: Mon, 19 Aug 2024 11:10:39 +0200 Subject: [PATCH 03/24] Fix tests --- packages/core/src/openapi-request-builder.ts | 2 +- ...hestration-completion-post-request.test.ts | 54 ++++++++----------- .../orchestration-filter-utility.test.ts | 29 +++++----- 3 files changed, 36 insertions(+), 49 deletions(-) diff --git a/packages/core/src/openapi-request-builder.ts b/packages/core/src/openapi-request-builder.ts index b6be5ee11..5a37cfc95 100644 --- a/packages/core/src/openapi-request-builder.ts +++ b/packages/core/src/openapi-request-builder.ts @@ -29,7 +29,7 @@ export class OpenApiRequestBuilder< // TODO: Remove explicit url! once we updated the type in the Cloud SDK, since url is always defined. return executeRequest( { url: url! }, - { deploymentConfiguration: {}, ...data }, + data, { ...rest } diff --git a/packages/gen-ai-hub/src/orchestration/orchestration-completion-post-request.test.ts b/packages/gen-ai-hub/src/orchestration/orchestration-completion-post-request.test.ts index 0f6568d90..515610a1d 100644 --- a/packages/gen-ai-hub/src/orchestration/orchestration-completion-post-request.test.ts +++ b/packages/gen-ai-hub/src/orchestration/orchestration-completion-post-request.test.ts @@ -1,30 +1,20 @@ import { CompletionPostRequest } from './client/api/index.js'; import { constructCompletionPostRequest } from './orchestration-client.js'; import { azureContentFilter } from './orchestration-filter-utility.js'; -import { GenAiHubCompletionParameters } from './orchestration-types.js'; +import { OrchestrationCompletionParameters } from './orchestration-types.js'; describe('constructCompletionPostRequest()', () => { - const genaihubCompletionParameters: GenAiHubCompletionParameters = { - deploymentConfiguration: { - deploymentId: 'deployment-id' - }, - llmConfig: { - model_name: 'gpt-35-turbo-16k', - model_params: { max_tokens: 50, temperature: 0.1 } - }, - prompt: { - template: [{ role: 'user', content: 'Hi' }] - } - }; + let input: OrchestrationCompletionParameters; beforeEach(() => { - genaihubCompletionParameters.llmConfig = { - model_name: 'gpt-35-turbo-16k', - model_params: { max_tokens: 50, temperature: 0.1 } - }; - genaihubCompletionParameters.prompt = { - template: [{ role: 'user', content: 'Hi' }] - }; + input = { + llmConfig: { + model_name: 'gpt-35-turbo-16k', + model_params: { max_tokens: 50, temperature: 0.1 } + }, prompt: { + template: [{ role: 'user', content: 'Hi' }] + } + } }); it('with model configuration and prompt template', async () => { @@ -42,12 +32,12 @@ describe('constructCompletionPostRequest()', () => { } }; const completionPostRequest: CompletionPostRequest = - constructCompletionPostRequest(genaihubCompletionParameters); + constructCompletionPostRequest(input); expect(completionPostRequest).toEqual(expectedCompletionPostRequest); }); it('with model configuration, prompt template and template params', async () => { - genaihubCompletionParameters.prompt = { + input.prompt = { template: [ { role: 'user', content: 'Create {number} paraphrases of {phrase}' } ], @@ -73,16 +63,16 @@ describe('constructCompletionPostRequest()', () => { input_params: { phrase: 'I hate you.', number: 3 } }; const completionPostRequest: CompletionPostRequest = - constructCompletionPostRequest(genaihubCompletionParameters); + constructCompletionPostRequest(input); expect(completionPostRequest).toEqual(expectedCompletionPostRequest); }); it('with model name, empty model parameters and prompt template', async () => { - genaihubCompletionParameters.llmConfig = { + input.llmConfig = { model_name: 'gpt-35-turbo-16k', model_params: {} }; - genaihubCompletionParameters.filterConfig = {}; + input.filterConfig = {}; const expectedCompletionPostRequest: CompletionPostRequest = { orchestration_config: { module_configurations: { @@ -102,12 +92,12 @@ describe('constructCompletionPostRequest()', () => { } }; const completionPostRequest: CompletionPostRequest = - constructCompletionPostRequest(genaihubCompletionParameters); + constructCompletionPostRequest(input); expect(completionPostRequest).toEqual(expectedCompletionPostRequest); }); it('with model configuration, prompt template and message history', async () => { - genaihubCompletionParameters.prompt = { + input.prompt = { template: [{ role: 'user', content: "What's my name?" }], messages_history: [ { @@ -161,12 +151,12 @@ describe('constructCompletionPostRequest()', () => { ] }; const completionPostRequest: CompletionPostRequest = - constructCompletionPostRequest(genaihubCompletionParameters); + constructCompletionPostRequest(input); expect(completionPostRequest).toEqual(expectedCompletionPostRequest); }); it('with model configuration, prompt template and filter configuration', async () => { - genaihubCompletionParameters.filterConfig = { + input.filterConfig = { input: azureContentFilter({ Hate: 4, SelfHarm: 0 }) }; const expectedCompletionPostRequest: CompletionPostRequest = { @@ -201,12 +191,12 @@ describe('constructCompletionPostRequest()', () => { } }; const completionPostRequest: CompletionPostRequest = - constructCompletionPostRequest(genaihubCompletionParameters); + constructCompletionPostRequest(input); expect(completionPostRequest).toEqual(expectedCompletionPostRequest); }); it('with model configuration, prompt template empty filter configuration', async () => { - genaihubCompletionParameters.filterConfig = {}; + input.filterConfig = {}; const expectedCompletionPostRequest: CompletionPostRequest = { orchestration_config: { module_configurations: { @@ -226,7 +216,7 @@ describe('constructCompletionPostRequest()', () => { } }; const completionPostRequest: CompletionPostRequest = - constructCompletionPostRequest(genaihubCompletionParameters); + constructCompletionPostRequest(input); expect(completionPostRequest).toEqual(expectedCompletionPostRequest); }); }); diff --git a/packages/gen-ai-hub/src/orchestration/orchestration-filter-utility.test.ts b/packages/gen-ai-hub/src/orchestration/orchestration-filter-utility.test.ts index 087837bd0..bcb665ae7 100644 --- a/packages/gen-ai-hub/src/orchestration/orchestration-filter-utility.test.ts +++ b/packages/gen-ai-hub/src/orchestration/orchestration-filter-utility.test.ts @@ -4,13 +4,10 @@ import { } from './client/api/index.js'; import { constructCompletionPostRequest } from './orchestration-client.js'; import { azureContentFilter } from './orchestration-filter-utility.js'; -import { GenAiHubCompletionParameters } from './orchestration-types.js'; +import { OrchestrationCompletionParameters } from './orchestration-types.js'; describe('Filter utility', () => { - const genaihubCompletionParameters: GenAiHubCompletionParameters = { - deploymentConfiguration: { - deploymentId: 'deployment-id' - }, + const input: OrchestrationCompletionParameters = { llmConfig: { model_name: 'gpt-35-turbo-16k', model_params: { max_tokens: 50, temperature: 0.1 } @@ -24,7 +21,7 @@ describe('Filter utility', () => { }; afterEach(() => { - genaihubCompletionParameters.filterConfig = undefined; + input.filterConfig = undefined; }); it('constructs filter configuration with only input', async () => { @@ -44,9 +41,9 @@ describe('Filter utility', () => { ] } }; - genaihubCompletionParameters.filterConfig = filterConfig; + input.filterConfig = filterConfig; const completionPostRequest: CompletionPostRequest = - constructCompletionPostRequest(genaihubCompletionParameters); + constructCompletionPostRequest(input); expect( completionPostRequest.orchestration_config.module_configurations .filtering_module_config @@ -70,9 +67,9 @@ describe('Filter utility', () => { ] } }; - genaihubCompletionParameters.filterConfig = filterConfig; + input.filterConfig = filterConfig; const completionPostRequest: CompletionPostRequest = - constructCompletionPostRequest(genaihubCompletionParameters); + constructCompletionPostRequest(input); expect( completionPostRequest.orchestration_config.module_configurations .filtering_module_config @@ -115,9 +112,9 @@ describe('Filter utility', () => { ] } }; - genaihubCompletionParameters.filterConfig = filterConfig; + input.filterConfig = filterConfig; const completionPostRequest: CompletionPostRequest = - constructCompletionPostRequest(genaihubCompletionParameters); + constructCompletionPostRequest(input); expect( completionPostRequest.orchestration_config.module_configurations .filtering_module_config @@ -129,9 +126,9 @@ describe('Filter utility', () => { input: azureContentFilter(), output: azureContentFilter() }; - genaihubCompletionParameters.filterConfig = filterConfig; + input.filterConfig = filterConfig; const completionPostRequest: CompletionPostRequest = - constructCompletionPostRequest(genaihubCompletionParameters); + constructCompletionPostRequest(input); const expectedFilterConfig: FilteringModuleConfig = { input: { filters: [ @@ -156,9 +153,9 @@ describe('Filter utility', () => { it('omits filter configuration if not set', async () => { const filterConfig: FilteringModuleConfig = {}; - genaihubCompletionParameters.filterConfig = filterConfig; + input.filterConfig = filterConfig; const completionPostRequest: CompletionPostRequest = - constructCompletionPostRequest(genaihubCompletionParameters); + constructCompletionPostRequest(input); expect( completionPostRequest.orchestration_config.module_configurations .filtering_module_config From 4732b39efa2966498e9557b37abe49c852a3927e Mon Sep 17 00:00:00 2001 From: cloud-sdk-js Date: Mon, 19 Aug 2024 09:11:19 +0000 Subject: [PATCH 04/24] fix: Changes from lint --- packages/core/src/openapi-request-builder.ts | 10 +++------- .../orchestration-completion-post-request.test.ts | 5 +++-- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/packages/core/src/openapi-request-builder.ts b/packages/core/src/openapi-request-builder.ts index 5a37cfc95..980ef526d 100644 --- a/packages/core/src/openapi-request-builder.ts +++ b/packages/core/src/openapi-request-builder.ts @@ -27,13 +27,9 @@ export class OpenApiRequestBuilder< async executeRaw(): Promise { const { url, data, ...rest } = await this.requestConfig(); // TODO: Remove explicit url! once we updated the type in the Cloud SDK, since url is always defined. - return executeRequest( - { url: url! }, - data, - { - ...rest - } - ); + return executeRequest({ url: url! }, data, { + ...rest + }); } /** diff --git a/packages/gen-ai-hub/src/orchestration/orchestration-completion-post-request.test.ts b/packages/gen-ai-hub/src/orchestration/orchestration-completion-post-request.test.ts index 515610a1d..c8a88a8f0 100644 --- a/packages/gen-ai-hub/src/orchestration/orchestration-completion-post-request.test.ts +++ b/packages/gen-ai-hub/src/orchestration/orchestration-completion-post-request.test.ts @@ -11,10 +11,11 @@ describe('constructCompletionPostRequest()', () => { llmConfig: { model_name: 'gpt-35-turbo-16k', model_params: { max_tokens: 50, temperature: 0.1 } - }, prompt: { + }, + prompt: { template: [{ role: 'user', content: 'Hi' }] } - } + }; }); it('with model configuration and prompt template', async () => { From 4458fe4bfce9af8508a2e8b420dc8ff3bffac803 Mon Sep 17 00:00:00 2001 From: Kuhr Date: Mon, 19 Aug 2024 11:11:43 +0200 Subject: [PATCH 05/24] Fix type test --- tests/type-tests/test/http-client.test-d.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/type-tests/test/http-client.test-d.ts b/tests/type-tests/test/http-client.test-d.ts index e8cb8b490..bbb3d5ec8 100644 --- a/tests/type-tests/test/http-client.test-d.ts +++ b/tests/type-tests/test/http-client.test-d.ts @@ -5,16 +5,8 @@ import { executeRequest } from '@sap-ai-sdk/core'; expectType>( executeRequest( { url: 'https://example.com', apiVersion: 'v1' }, - { deploymentConfiguration: { deploymentId: 'id' }, prompt: 'test prompt' }, { headers: { 'Content-Type': 'application/json' } } ) ); -expectError( - executeRequest( - { url: 'https://example.com', apiVersion: 'v1' }, - { prompt: 'test prompt' } - ) -); - expectError(executeRequest({}, { prompt: 'test prompt' })); From b68ff8d4c637835ffea3c4a33dac2061a67d62d7 Mon Sep 17 00:00:00 2001 From: Kuhr Date: Mon, 19 Aug 2024 15:26:59 +0200 Subject: [PATCH 06/24] Add alternative API variant --- .../src/client/openai/openai-types.ts | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/packages/gen-ai-hub/src/client/openai/openai-types.ts b/packages/gen-ai-hub/src/client/openai/openai-types.ts index 1b0b8e734..071eb4f4b 100644 --- a/packages/gen-ai-hub/src/client/openai/openai-types.ts +++ b/packages/gen-ai-hub/src/client/openai/openai-types.ts @@ -1,7 +1,6 @@ // TODO: docs /* eslint-disable */ export const OpenAiModels = { - // TODO: figure out if these should be constants or functions, e.g. to allow for dynamic versioning GPT_4o: { name: 'gpt-4o', type: 'chat', @@ -45,6 +44,18 @@ export const OpenAiModels = { } as OpenAiEmbeddingModel }; +// alternative to OpenAiModels +export class OpenAiModels2 { + static GPT_4o(version: 'latest' | '0613' = 'latest'): OpenAiChatModel { + return { + name: 'gpt-4o', + type: 'chat', + version: version + }; + } + private constructor() {} +} + export interface OpenAiChatModel { name: | 'gpt-4o' @@ -53,14 +64,16 @@ export interface OpenAiChatModel { | 'gpt-35-turbo' | 'gpt-35-turbo-0125' | 'gpt-35-turbo-16k'; - version: 'latest'; + version: string; + type: 'chat'; } export interface OpenAiEmbeddingModel { name: | 'text-embedding-ada-002' | 'text-embedding-3-small' | 'text-embedding-3-large'; - version: 'latest'; + version: string; + type: 'embedding'; } /** From a9671a178fd3b6c3c1e22d588b1c131512f08691 Mon Sep 17 00:00:00 2001 From: Kuhr Date: Mon, 19 Aug 2024 17:31:13 +0200 Subject: [PATCH 07/24] Make models readonly --- packages/gen-ai-hub/src/client/openai/openai-types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gen-ai-hub/src/client/openai/openai-types.ts b/packages/gen-ai-hub/src/client/openai/openai-types.ts index 071eb4f4b..78bf9da45 100644 --- a/packages/gen-ai-hub/src/client/openai/openai-types.ts +++ b/packages/gen-ai-hub/src/client/openai/openai-types.ts @@ -42,7 +42,7 @@ export const OpenAiModels = { type: 'embedding', version: 'latest' } as OpenAiEmbeddingModel -}; +} as const; // alternative to OpenAiModels export class OpenAiModels2 { From dd676c0e31ffeebe6b5ab3ab945d57bf1142db12 Mon Sep 17 00:00:00 2001 From: Kuhr Date: Tue, 20 Aug 2024 09:30:48 +0200 Subject: [PATCH 08/24] Remove obsolete class --- packages/gen-ai-hub/src/client/interface.ts | 28 --------------------- 1 file changed, 28 deletions(-) delete mode 100644 packages/gen-ai-hub/src/client/interface.ts diff --git a/packages/gen-ai-hub/src/client/interface.ts b/packages/gen-ai-hub/src/client/interface.ts deleted file mode 100644 index d67e82de6..000000000 --- a/packages/gen-ai-hub/src/client/interface.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { BaseLlmParameters, CustomRequestConfig } from '@sap-ai-sdk/core'; -import { BaseLlmOutput } from './types.js'; - -/** - * The base client interface for all provider specific clients. - */ -export interface BaseClient { - /** - * Creates a completion for the chat messages. - * @param data - The input parameters for the chat completion. - * @param requestConfig - The request configuration. - * @returns The completion result. - */ - chatCompletion( - data: T, - requestConfig?: CustomRequestConfig - ): Promise; - /** - * Creates an embedding vector representing the given text. - * @param data - The input parameters for the chat completion. - * @param requestConfig - The request configuration. - * @returns The completion result. - */ - embeddings( - data: T, - requestConfig?: CustomRequestConfig - ): Promise; -} From 41ae69e5aabe213cec70171495a30d89666d3ce7 Mon Sep 17 00:00:00 2001 From: Kuhr Date: Thu, 22 Aug 2024 09:08:25 +0200 Subject: [PATCH 09/24] Fix tests --- .../src/client/openai/openai-client.ts | 2 +- packages/gen-ai-hub/src/core.test.ts | 78 ---------- .../src/orchestration/orchestration-client.ts | 2 +- .../src/utils/deployment-resolver.test.ts | 133 ++++++++++++++++++ .../{core.ts => utils/deployment-resolver.ts} | 44 +++--- 5 files changed, 154 insertions(+), 105 deletions(-) delete mode 100644 packages/gen-ai-hub/src/core.test.ts create mode 100644 packages/gen-ai-hub/src/utils/deployment-resolver.test.ts rename packages/gen-ai-hub/src/{core.ts => utils/deployment-resolver.ts} (61%) diff --git a/packages/gen-ai-hub/src/client/openai/openai-client.ts b/packages/gen-ai-hub/src/client/openai/openai-client.ts index 838ddfb00..30d6b3254 100644 --- a/packages/gen-ai-hub/src/client/openai/openai-client.ts +++ b/packages/gen-ai-hub/src/client/openai/openai-client.ts @@ -1,6 +1,6 @@ import { HttpRequestConfig } from '@sap-cloud-sdk/http-client'; import { CustomRequestConfig, executeRequest } from '@sap-ai-sdk/core'; -import { DeploymentResolver, resolveDeployment } from '../../core.js'; +import { DeploymentResolver, resolveDeployment } from '../../utils/deployment-resolver.js'; import { OpenAiChatCompletionParameters, OpenAiEmbeddingParameters, diff --git a/packages/gen-ai-hub/src/core.test.ts b/packages/gen-ai-hub/src/core.test.ts deleted file mode 100644 index f2eb69ccd..000000000 --- a/packages/gen-ai-hub/src/core.test.ts +++ /dev/null @@ -1,78 +0,0 @@ -import nock from 'nock'; -import { - mockClientCredentialsGrantCall, - mockAiCoreEnvVariable, - aiCoreDestination -} from '../../../test-util/mock-http.js'; -import { resolveDeployment } from './core.js'; - -describe('AICore', () => { - beforeEach(() => { - mockAiCoreEnvVariable(); - mockClientCredentialsGrantCall(); - }); - - afterEach(() => { - nock.cleanAll(); - }); - - it('should be defined', async () => { - nock(aiCoreDestination.url, { - reqheaders: { - 'ai-resource-group': 'default' - } - }) - .get('/v2/lm/deployments') - .query({ scenarioId: 'foundation-models', status: 'RUNNING' }) - .reply(200, { - count: 1, - resources: [ - { - configurationId: 'af7a5804-0820-4fbb-8e09-04837b204095', - configurationName: 'gpt-4-32k', - deploymentUrl: - 'https://api.ai.staging.eu-west-1.mlf-aws-dev.com/v2/inference/deployments/d0d49e445e7df086', - details: { - resources: { - backend_details: { - model: { - name: 'gpt-4-32k', - version: 'latest' - } - } - }, - scaling: { - backend_details: {} - } - }, - id: 'd0d49e445e7df086', - lastOperation: 'CREATE', - status: 'RUNNING' - } - ] - }); - - const result = await resolveDeployment({ scenarioId: 'foundation-models' }); - expect(result).toBeDefined(); - expect(result.id).toBe('d0d49e445e7df086'); - expect(result.configurationName).toBe('gpt-4-32k'); - }); - - it('should throw on empty list', async () => { - nock(aiCoreDestination.url, { - reqheaders: { - 'ai-resource-group': 'default' - } - }) - .get('/v2/lm/deployments') - .query({ scenarioId: 'foundation-models', status: 'RUNNING' }) - .reply(200, { - count: 0, - resources: [] - }); - - await expect( - resolveDeployment({ scenarioId: 'foundation-models' }) - ).rejects.toThrow('No deployment matched the given criteria'); - }); -}); diff --git a/packages/gen-ai-hub/src/orchestration/orchestration-client.ts b/packages/gen-ai-hub/src/orchestration/orchestration-client.ts index f3abf46a0..63ca07a66 100644 --- a/packages/gen-ai-hub/src/orchestration/orchestration-client.ts +++ b/packages/gen-ai-hub/src/orchestration/orchestration-client.ts @@ -1,5 +1,5 @@ import { executeRequest, CustomRequestConfig } from '@sap-ai-sdk/core'; -import { DeploymentResolver, resolveDeployment } from '../core.js'; +import { DeploymentResolver, resolveDeployment } from '../utils/deployment-resolver.js'; import { CompletionPostRequest, CompletionPostResponse diff --git a/packages/gen-ai-hub/src/utils/deployment-resolver.test.ts b/packages/gen-ai-hub/src/utils/deployment-resolver.test.ts new file mode 100644 index 000000000..0b4583a67 --- /dev/null +++ b/packages/gen-ai-hub/src/utils/deployment-resolver.test.ts @@ -0,0 +1,133 @@ +import nock from 'nock'; +import { + mockClientCredentialsGrantCall, + mockAiCoreEnvVariable, + aiCoreDestination +} from '../../../../test-util/mock-http.js'; +import { resolveDeployment } from './deployment-resolver.js'; + +describe('Deployment resolver', () => { + beforeEach(() => { + mockAiCoreEnvVariable(); + mockClientCredentialsGrantCall(); + }); + + afterEach(() => { + nock.cleanAll(); + }); + + describe('should lookup the deployment ID based on a scenario', () => { + beforeEach(() => { + mockResponse(); + }); + it('should return the first deployment, if multiple are given', async () => { + const { id, configurationId } = await resolveDeployment({ + scenarioId: 'foundation-models' + }); + expect(id).toBe('1'); + expect(configurationId).toBe('c1'); + }); + it('should return the deployment with the correct model name', async () => { + const { id, configurationId } = await resolveDeployment({ + scenarioId: 'foundation-models', + modelName: 'gpt-4o' + }); + expect(id).toBe('2'); + expect(configurationId).toBe('c2'); + }); + it('should return the deployment with the correct model name', async () => { + const { id, configurationId } = await resolveDeployment({ + scenarioId: 'foundation-models', + modelVersion: '0613' + }); + expect(id).toBe('2'); + expect(configurationId).toBe('c2'); + }); + it('should throw in case no deployment with the given model name is found', async () => { + await expect( + resolveDeployment({ + scenarioId: 'foundation-models', + modelName: 'not existing' + }) + ).rejects.toThrow('No deployment matched the given criteria'); + }); + it('should throw in case no deployment with the given model version is found', async () => { + await expect( + resolveDeployment({ + scenarioId: 'foundation-models', + modelName: 'gpt-4o', + modelVersion: 'not existing' + }) + ).rejects.toThrow('No deployment matched the given criteria'); + }); + }); + + it('should throw on empty list', async () => { + nock(aiCoreDestination.url, { + reqheaders: { + 'ai-resource-group': 'default' + } + }) + .get('/v2/lm/deployments') + .query({ scenarioId: 'foundation-models', status: 'RUNNING' }) + .reply(200, { + count: 0, + resources: [] + }); + + await expect( + resolveDeployment({ scenarioId: 'foundation-models' }) + ).rejects.toThrow('No deployment matched the given criteria'); + }); +}); + +function mockResponse() { + nock(aiCoreDestination.url, { + reqheaders: { + 'ai-resource-group': 'default' + } + }) + .get('/v2/lm/deployments') + .query({ scenarioId: 'foundation-models', status: 'RUNNING' }) + .reply(200, { + count: 1, + resources: [ + { + configurationId: 'c1', + id: '1', + deploymentUrl: 'https://foo.com/v2/inference/deployments/1', + details: { + resources: { + backend_details: { + model: { + name: 'gpt-4-32k', + version: 'latest' + } + } + }, + scaling: { + backend_details: {} + } + }, + lastOperation: 'CREATE', + status: 'RUNNING' + }, + { + configurationId: 'c2', + id: '2', + deploymentUrl: 'https://foo.com/v2/inference/deployments/2', + details: { + resources: { + backend_details: { + model: { + name: 'gpt-4o', + version: '0613' + } + } + } + }, + status: 'RUNNING' + } + ] + }); +} diff --git a/packages/gen-ai-hub/src/core.ts b/packages/gen-ai-hub/src/utils/deployment-resolver.ts similarity index 61% rename from packages/gen-ai-hub/src/core.ts rename to packages/gen-ai-hub/src/utils/deployment-resolver.ts index ca2943444..69a7033d7 100644 --- a/packages/gen-ai-hub/src/core.ts +++ b/packages/gen-ai-hub/src/utils/deployment-resolver.ts @@ -1,4 +1,8 @@ -import { DeploymentApi, AiDeployment } from '@sap-ai-sdk/ai-core'; +import { + DeploymentApi, + AiDeployment, + AiDeploymentStatus +} from '@sap-ai-sdk/ai-core'; // TODO: docs /* eslint-disable */ @@ -21,45 +25,35 @@ export async function resolveDeployment(opts: { modelName?: string; modelVersion?: string; }): Promise { - // TODO: is there a more elegant way to write this in TS? - let query: any; - if (opts.executableId) { - query = { - scenarioId: opts.scenarioId, - status: 'RUNNING', - executableIds: [opts.executableId] - }; - } else { - query = { scenarioId: opts.scenarioId, status: 'RUNNING' }; - } + const query = { + scenarioId: opts.scenarioId, + status: 'RUNNING' as AiDeploymentStatus, + ...(opts.executableId && { executableIds: [opts.executableId] }) + }; - // TODO: add a cache + // TODO: add a cache: https://github.tools.sap/AI/gen-ai-hub-sdk-js-backlog/issues/78 let deploymentList: AiDeployment[]; + const { deploymentQuery } = DeploymentApi; + const resourceGroup = {'AI-Resource-Group': 'default'}; try { - deploymentList = await DeploymentApi.deploymentQuery(query, { - 'AI-Resource-Group': 'default' - }) - .execute() - .then(res => res.resources); + deploymentList = (await deploymentQuery(query,resourceGroup).execute()).resources; } catch (error) { throw new Error('Failed to fetch the list of deployments: ' + error); } if (opts.modelName) { deploymentList = deploymentList.filter( - (deployment: any) => - modelExtractor(deployment)?.modelName === opts.modelName + deployment => modelExtractor(deployment)?.name === opts.modelName ); } if (opts.modelVersion) { - // feature idea: smart handling of 'latest' version + // feature idea: smart handling of 'latest' version: treat 'latest' and the highest version number as the same deploymentList = deploymentList.filter( - (deployment: any) => - modelExtractor(deployment)?.modelVersion === opts.modelVersion + deployment => + modelExtractor(deployment)?.version === opts.modelVersion ); } if (deploymentList.length === 0) { - // TODO: return undefined instead? throw new Error( 'No deployment matched the given criteria: ' + JSON.stringify(opts) ); @@ -67,5 +61,5 @@ export async function resolveDeployment(opts: { return deploymentList[0]; } -const modelExtractor = (deployment: any) => +const modelExtractor = (deployment: AiDeployment) => deployment.details?.resources?.backend_details?.model; From 3b2c87d869a1b7afc2a41075f654b3016a8e7a54 Mon Sep 17 00:00:00 2001 From: cloud-sdk-js Date: Thu, 22 Aug 2024 07:09:00 +0000 Subject: [PATCH 10/24] fix: Changes from lint --- packages/gen-ai-hub/src/client/openai/openai-client.ts | 5 ++++- .../gen-ai-hub/src/orchestration/orchestration-client.ts | 5 ++++- packages/gen-ai-hub/src/utils/deployment-resolver.ts | 8 ++++---- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/gen-ai-hub/src/client/openai/openai-client.ts b/packages/gen-ai-hub/src/client/openai/openai-client.ts index 30d6b3254..de9d4b315 100644 --- a/packages/gen-ai-hub/src/client/openai/openai-client.ts +++ b/packages/gen-ai-hub/src/client/openai/openai-client.ts @@ -1,6 +1,9 @@ import { HttpRequestConfig } from '@sap-cloud-sdk/http-client'; import { CustomRequestConfig, executeRequest } from '@sap-ai-sdk/core'; -import { DeploymentResolver, resolveDeployment } from '../../utils/deployment-resolver.js'; +import { + DeploymentResolver, + resolveDeployment +} from '../../utils/deployment-resolver.js'; import { OpenAiChatCompletionParameters, OpenAiEmbeddingParameters, diff --git a/packages/gen-ai-hub/src/orchestration/orchestration-client.ts b/packages/gen-ai-hub/src/orchestration/orchestration-client.ts index 63ca07a66..c8391d081 100644 --- a/packages/gen-ai-hub/src/orchestration/orchestration-client.ts +++ b/packages/gen-ai-hub/src/orchestration/orchestration-client.ts @@ -1,5 +1,8 @@ import { executeRequest, CustomRequestConfig } from '@sap-ai-sdk/core'; -import { DeploymentResolver, resolveDeployment } from '../utils/deployment-resolver.js'; +import { + DeploymentResolver, + resolveDeployment +} from '../utils/deployment-resolver.js'; import { CompletionPostRequest, CompletionPostResponse diff --git a/packages/gen-ai-hub/src/utils/deployment-resolver.ts b/packages/gen-ai-hub/src/utils/deployment-resolver.ts index 69a7033d7..c835dd2ba 100644 --- a/packages/gen-ai-hub/src/utils/deployment-resolver.ts +++ b/packages/gen-ai-hub/src/utils/deployment-resolver.ts @@ -34,9 +34,10 @@ export async function resolveDeployment(opts: { // TODO: add a cache: https://github.tools.sap/AI/gen-ai-hub-sdk-js-backlog/issues/78 let deploymentList: AiDeployment[]; const { deploymentQuery } = DeploymentApi; - const resourceGroup = {'AI-Resource-Group': 'default'}; + const resourceGroup = { 'AI-Resource-Group': 'default' }; try { - deploymentList = (await deploymentQuery(query,resourceGroup).execute()).resources; + deploymentList = (await deploymentQuery(query, resourceGroup).execute()) + .resources; } catch (error) { throw new Error('Failed to fetch the list of deployments: ' + error); } @@ -49,8 +50,7 @@ export async function resolveDeployment(opts: { if (opts.modelVersion) { // feature idea: smart handling of 'latest' version: treat 'latest' and the highest version number as the same deploymentList = deploymentList.filter( - deployment => - modelExtractor(deployment)?.version === opts.modelVersion + deployment => modelExtractor(deployment)?.version === opts.modelVersion ); } if (deploymentList.length === 0) { From 5f6c351bb931636ded34d4182029f9d8de3bdbaa Mon Sep 17 00:00:00 2001 From: Marika Marszalkowski Date: Thu, 22 Aug 2024 14:13:01 +0200 Subject: [PATCH 11/24] add alternative proposal --- .../src/client/openai/openai-types.ts | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/packages/gen-ai-hub/src/client/openai/openai-types.ts b/packages/gen-ai-hub/src/client/openai/openai-types.ts index 78bf9da45..3a96607b0 100644 --- a/packages/gen-ai-hub/src/client/openai/openai-types.ts +++ b/packages/gen-ai-hub/src/client/openai/openai-types.ts @@ -56,6 +56,40 @@ export class OpenAiModels2 { private constructor() {} } +/** + * + * Alternative proposal for referencing models. + * + */ +export type OpenAiChatModelMM = + | 'gpt-4o' + | 'gpt-4' + | 'gpt-4-32k' + | 'gpt-35-turbo' + | 'gpt-35-turbo-0125' + | 'gpt-35-turbo-16k'; + +export type OpenAiEmbeddingModelMM = + | 'text-embedding-ada-002' + | 'text-embedding-3-small' + | 'text-embedding-3-large'; + +// example chat completion API with version +function chatCompletion( + model: OpenAiChatModelMM | { name: OpenAiChatModelMM; version: string } +) {} + +//example usage with default 'latest' version +chatCompletion('gpt-4o'); + +//example usage with specific version +chatCompletion({ name: 'gpt-4o', version: '2024-08-22' }); +/** + * + * End alternative proposal. + * + */ + export interface OpenAiChatModel { name: | 'gpt-4o' From 8518347d4a44cfef994edda4ec6d71f4233e1478 Mon Sep 17 00:00:00 2001 From: Kuhr Date: Thu, 22 Aug 2024 14:50:57 +0200 Subject: [PATCH 12/24] Minor improvements --- .../src/client/openai/openai-types.ts | 2 +- packages/gen-ai-hub/src/client/types.ts | 9 ------ packages/gen-ai-hub/src/index.ts | 9 +----- .../src/utils/deployment-resolver.test.ts | 2 -- sample-code/src/aiservice.ts | 31 ++++++++++++------- 5 files changed, 21 insertions(+), 32 deletions(-) delete mode 100644 packages/gen-ai-hub/src/client/types.ts diff --git a/packages/gen-ai-hub/src/client/openai/openai-types.ts b/packages/gen-ai-hub/src/client/openai/openai-types.ts index 3a96607b0..9d6e40a4e 100644 --- a/packages/gen-ai-hub/src/client/openai/openai-types.ts +++ b/packages/gen-ai-hub/src/client/openai/openai-types.ts @@ -58,7 +58,7 @@ export class OpenAiModels2 { /** * - * Alternative proposal for referencing models. + * Alternative proposal for referencing models.OpenAiChatAssistantMessage * */ export type OpenAiChatModelMM = diff --git a/packages/gen-ai-hub/src/client/types.ts b/packages/gen-ai-hub/src/client/types.ts deleted file mode 100644 index 42aa1caa1..000000000 --- a/packages/gen-ai-hub/src/client/types.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { - OpenAiEmbeddingOutput, - OpenAiChatCompletionOutput -} from './openai/openai-types.js'; - -/** - * Base LLM Output. - */ -export type BaseLlmOutput = OpenAiChatCompletionOutput | OpenAiEmbeddingOutput; diff --git a/packages/gen-ai-hub/src/index.ts b/packages/gen-ai-hub/src/index.ts index d4859e10c..d9ecfe895 100644 --- a/packages/gen-ai-hub/src/index.ts +++ b/packages/gen-ai-hub/src/index.ts @@ -1,11 +1,4 @@ -export { - OpenAiClient, - OpenAiChatCompletionParameters, - OpenAiEmbeddingParameters, - OpenAiEmbeddingOutput, - OpenAiChatCompletionOutput, - OpenAiModels -} from './client/index.js'; +export * from './client/index.js'; export { OrchestrationClient, OrchestrationCompletionParameters, diff --git a/packages/gen-ai-hub/src/utils/deployment-resolver.test.ts b/packages/gen-ai-hub/src/utils/deployment-resolver.test.ts index 0b4583a67..cc14a80fa 100644 --- a/packages/gen-ai-hub/src/utils/deployment-resolver.test.ts +++ b/packages/gen-ai-hub/src/utils/deployment-resolver.test.ts @@ -1,14 +1,12 @@ import nock from 'nock'; import { mockClientCredentialsGrantCall, - mockAiCoreEnvVariable, aiCoreDestination } from '../../../../test-util/mock-http.js'; import { resolveDeployment } from './deployment-resolver.js'; describe('Deployment resolver', () => { beforeEach(() => { - mockAiCoreEnvVariable(); mockClientCredentialsGrantCall(); }); diff --git a/sample-code/src/aiservice.ts b/sample-code/src/aiservice.ts index ba540f1e3..0fed80086 100644 --- a/sample-code/src/aiservice.ts +++ b/sample-code/src/aiservice.ts @@ -1,4 +1,8 @@ -import { OpenAiClient, OpenAiModels } from '@sap-ai-sdk/gen-ai-hub'; +import { + OpenAiClient, + OpenAiModels, + OpenAiChatAssistantMessage +} from '@sap-ai-sdk/gen-ai-hub'; const openAiClient = new OpenAiClient(); @@ -6,22 +10,25 @@ const openAiClient = new OpenAiClient(); * Ask GPT about the capital of France. * @returns The answer from GPT. */ -export function chatCompletion(): Promise { - return openAiClient - .chatCompletion(OpenAiModels.GPT_35_TURBO, { +export async function chatCompletion(): Promise { + const response = await openAiClient.chatCompletion( + OpenAiModels.GPT_35_TURBO, + { messages: [{ role: 'user', content: 'What is the capital of France?' }] - }) - .then(response => response.choices[0].message.content); + } + ); + const assistantMessage = response.choices[0] + .message as OpenAiChatAssistantMessage; + return assistantMessage.content!; } /** * Embed 'Hello, world!' using the OpenAI ADA model. * @returns An embedding vector. */ -export function computeEmbedding(): Promise { - return openAiClient - .embeddings(OpenAiModels.ADA_002, { - input: 'Hello, world!' - }) - .then(response => response.data[0].embedding); +export async function computeEmbedding(): Promise { + const response = await openAiClient.embeddings(OpenAiModels.ADA_002, { + input: 'Hello, world!' + }); + return response.data[0].embedding; } From a745b8c1eb7c31399c2b8c6936cbe0b101f5d2cc Mon Sep 17 00:00:00 2001 From: Kuhr Date: Thu, 22 Aug 2024 16:17:01 +0200 Subject: [PATCH 13/24] Refactoring from API discussion --- .../src/client/openai/openai-client.test.ts | 9 +- .../src/client/openai/openai-client.ts | 38 ++++--- .../src/client/openai/openai-types.ts | 100 +----------------- .../src/utils/deployment-resolver.test.ts | 9 +- .../src/utils/deployment-resolver.ts | 23 ++-- sample-code/src/aiservice.ts | 8 +- 6 files changed, 44 insertions(+), 143 deletions(-) diff --git a/packages/gen-ai-hub/src/client/openai/openai-client.test.ts b/packages/gen-ai-hub/src/client/openai/openai-client.test.ts index 3d1032c28..3a07cc7e4 100644 --- a/packages/gen-ai-hub/src/client/openai/openai-client.test.ts +++ b/packages/gen-ai-hub/src/client/openai/openai-client.test.ts @@ -9,7 +9,6 @@ import { OpenAiChatMessage, OpenAiEmbeddingOutput, OpenAiEmbeddingParameters, - OpenAiModels } from './openai-types.js'; import { OpenAiClient } from './openai-client.js'; @@ -61,7 +60,7 @@ describe('openai client', () => { ); const response = await client.chatCompletion( - OpenAiModels.GPT_4o, + 'gpt-35-turbo', prompt, '1234' ); @@ -87,7 +86,7 @@ describe('openai client', () => { ); await expect( - client.chatCompletion(OpenAiModels.GPT_4o, prompt, '1234') + client.chatCompletion('gpt-4', prompt, '1234') ).rejects.toThrow('status code 400'); }); }); @@ -113,7 +112,7 @@ describe('openai client', () => { embeddingsEndpoint ); const response = await client.embeddings( - OpenAiModels.ADA_002, + 'text-embedding-ada-002', prompt, '1234' ); @@ -139,7 +138,7 @@ describe('openai client', () => { ); await expect( - client.embeddings(OpenAiModels.ADA_002, prompt, '1234') + client.embeddings('text-embedding-3-large', prompt, '1234') ).rejects.toThrow('status code 400'); }); }); diff --git a/packages/gen-ai-hub/src/client/openai/openai-client.ts b/packages/gen-ai-hub/src/client/openai/openai-client.ts index de9d4b315..6c07f5f1c 100644 --- a/packages/gen-ai-hub/src/client/openai/openai-client.ts +++ b/packages/gen-ai-hub/src/client/openai/openai-client.ts @@ -2,6 +2,7 @@ import { HttpRequestConfig } from '@sap-cloud-sdk/http-client'; import { CustomRequestConfig, executeRequest } from '@sap-ai-sdk/core'; import { DeploymentResolver, + FoundationModel, resolveDeployment } from '../../utils/deployment-resolver.js'; import { @@ -28,15 +29,12 @@ export class OpenAiClient { * @returns The completion result. */ async chatCompletion( - model: OpenAiChatModel, + model: OpenAiChatModel | { name: OpenAiChatModel; version: string }, data: OpenAiChatCompletionParameters, - deploymentResolver: DeploymentResolver = getDeploymentResolver(model), + deploymentResolver?: DeploymentResolver, requestConfig?: CustomRequestConfig ): Promise { - const deployment = - typeof deploymentResolver === 'function' - ? (await deploymentResolver()).id - : deploymentResolver; + const deployment = await resolveOpenAiDeployment(model, deploymentResolver); const response = await executeRequest( { url: `/inference/deployments/${deployment}/chat/completions`, @@ -56,15 +54,12 @@ export class OpenAiClient { * @returns The completion result. */ async embeddings( - model: OpenAiEmbeddingModel, + model: OpenAiEmbeddingModel | { name: OpenAiEmbeddingModel; version: string }, data: OpenAiEmbeddingParameters, - deploymentResolver: DeploymentResolver = getDeploymentResolver(model), + deploymentResolver?: DeploymentResolver, requestConfig?: CustomRequestConfig ): Promise { - const deployment = - typeof deploymentResolver === 'function' - ? (await deploymentResolver()).id - : deploymentResolver; + const deployment = await resolveOpenAiDeployment(model, deploymentResolver); const response = await executeRequest( { url: `/inference/deployments/${deployment}/embeddings`, apiVersion }, data, @@ -85,12 +80,15 @@ export class OpenAiClient { } } -function getDeploymentResolver(model: OpenAiChatModel | OpenAiEmbeddingModel) { - return () => - resolveDeployment({ - scenarioId: 'foundation-models', - executableId: 'azure-openai', - modelName: model.name, - modelVersion: model.version - }); +async function resolveOpenAiDeployment(model: string | { name: string; version: string }, resolver?: DeploymentResolver) { + if (typeof resolver === 'string') { + return resolver; + } + const llm = typeof model === 'string' ? { name: model, version: 'latest' } : model; + return (await resolveDeployment({ + scenarioId: 'foundation-models', + executableId: 'azure-openai', + model: llm + }) + ).id; } diff --git a/packages/gen-ai-hub/src/client/openai/openai-types.ts b/packages/gen-ai-hub/src/client/openai/openai-types.ts index 9d6e40a4e..cb032b9ab 100644 --- a/packages/gen-ai-hub/src/client/openai/openai-types.ts +++ b/packages/gen-ai-hub/src/client/openai/openai-types.ts @@ -1,67 +1,7 @@ // TODO: docs /* eslint-disable */ -export const OpenAiModels = { - GPT_4o: { - name: 'gpt-4o', - type: 'chat', - version: 'latest' - } as OpenAiChatModel, - GPT_4: { name: 'gpt-4', type: 'chat', version: 'latest' } as OpenAiChatModel, - GPT_4_32K: { - name: 'gpt-4-32k', - type: 'chat', - version: 'latest' - } as OpenAiChatModel, - GPT_35_TURBO: { - name: 'gpt-35-turbo', - type: 'chat', - version: 'latest' - } as OpenAiChatModel, - GPT_35_TURBO_0125: { - name: 'gpt-35-turbo-0125', - type: 'chat', - version: 'latest' - } as OpenAiChatModel, - GPT_35_TURBO_16K: { - name: 'gpt-35-turbo-16k', - type: 'chat', - version: 'latest' - } as OpenAiChatModel, - ADA_002: { - name: 'text-embedding-ada-002', - type: 'embedding', - version: 'latest' - } as OpenAiEmbeddingModel, - TEXT_EMBEDDING_3_SMALL: { - name: 'text-embedding-3-small', - type: 'embedding', - version: 'latest' - } as OpenAiEmbeddingModel, - TEXT_EMBEDDING_3_LARGE: { - name: 'text-embedding-3-large', - type: 'embedding', - version: 'latest' - } as OpenAiEmbeddingModel -} as const; - -// alternative to OpenAiModels -export class OpenAiModels2 { - static GPT_4o(version: 'latest' | '0613' = 'latest'): OpenAiChatModel { - return { - name: 'gpt-4o', - type: 'chat', - version: version - }; - } - private constructor() {} -} -/** - * - * Alternative proposal for referencing models.OpenAiChatAssistantMessage - * - */ -export type OpenAiChatModelMM = +export type OpenAiChatModel = | 'gpt-4o' | 'gpt-4' | 'gpt-4-32k' @@ -69,47 +9,11 @@ export type OpenAiChatModelMM = | 'gpt-35-turbo-0125' | 'gpt-35-turbo-16k'; -export type OpenAiEmbeddingModelMM = +export type OpenAiEmbeddingModel = | 'text-embedding-ada-002' | 'text-embedding-3-small' | 'text-embedding-3-large'; -// example chat completion API with version -function chatCompletion( - model: OpenAiChatModelMM | { name: OpenAiChatModelMM; version: string } -) {} - -//example usage with default 'latest' version -chatCompletion('gpt-4o'); - -//example usage with specific version -chatCompletion({ name: 'gpt-4o', version: '2024-08-22' }); -/** - * - * End alternative proposal. - * - */ - -export interface OpenAiChatModel { - name: - | 'gpt-4o' - | 'gpt-4' - | 'gpt-4-32k' - | 'gpt-35-turbo' - | 'gpt-35-turbo-0125' - | 'gpt-35-turbo-16k'; - version: string; - type: 'chat'; -} -export interface OpenAiEmbeddingModel { - name: - | 'text-embedding-ada-002' - | 'text-embedding-3-small' - | 'text-embedding-3-large'; - version: string; - type: 'embedding'; -} - /** * OpenAI system message. */ diff --git a/packages/gen-ai-hub/src/utils/deployment-resolver.test.ts b/packages/gen-ai-hub/src/utils/deployment-resolver.test.ts index cc14a80fa..177f6b9b1 100644 --- a/packages/gen-ai-hub/src/utils/deployment-resolver.test.ts +++ b/packages/gen-ai-hub/src/utils/deployment-resolver.test.ts @@ -28,7 +28,7 @@ describe('Deployment resolver', () => { it('should return the deployment with the correct model name', async () => { const { id, configurationId } = await resolveDeployment({ scenarioId: 'foundation-models', - modelName: 'gpt-4o' + model: {name: 'gpt-4o'} }); expect(id).toBe('2'); expect(configurationId).toBe('c2'); @@ -36,7 +36,7 @@ describe('Deployment resolver', () => { it('should return the deployment with the correct model name', async () => { const { id, configurationId } = await resolveDeployment({ scenarioId: 'foundation-models', - modelVersion: '0613' + model: {name: 'gpt-4o', version: '0613'} }); expect(id).toBe('2'); expect(configurationId).toBe('c2'); @@ -45,7 +45,7 @@ describe('Deployment resolver', () => { await expect( resolveDeployment({ scenarioId: 'foundation-models', - modelName: 'not existing' + model: {name: 'not existing'} }) ).rejects.toThrow('No deployment matched the given criteria'); }); @@ -53,8 +53,7 @@ describe('Deployment resolver', () => { await expect( resolveDeployment({ scenarioId: 'foundation-models', - modelName: 'gpt-4o', - modelVersion: 'not existing' + model: {name: 'gpt-4o', version: 'not existing'} }) ).rejects.toThrow('No deployment matched the given criteria'); }); diff --git a/packages/gen-ai-hub/src/utils/deployment-resolver.ts b/packages/gen-ai-hub/src/utils/deployment-resolver.ts index c835dd2ba..3cc1fd776 100644 --- a/packages/gen-ai-hub/src/utils/deployment-resolver.ts +++ b/packages/gen-ai-hub/src/utils/deployment-resolver.ts @@ -9,6 +9,7 @@ import { export type DeploymentResolver = DeploymentId | (() => Promise); export type DeploymentId = string; +export type FoundationModel = { name: string; version?: string }; // TODO: figure out what the best search criteria are /** @@ -22,8 +23,7 @@ export type DeploymentId = string; export async function resolveDeployment(opts: { scenarioId: string; executableId?: string; - modelName?: string; - modelVersion?: string; + model?: FoundationModel; }): Promise { const query = { scenarioId: opts.scenarioId, @@ -42,17 +42,18 @@ export async function resolveDeployment(opts: { throw new Error('Failed to fetch the list of deployments: ' + error); } - if (opts.modelName) { + if (opts.model) { deploymentList = deploymentList.filter( - deployment => modelExtractor(deployment)?.name === opts.modelName - ); - } - if (opts.modelVersion) { - // feature idea: smart handling of 'latest' version: treat 'latest' and the highest version number as the same - deploymentList = deploymentList.filter( - deployment => modelExtractor(deployment)?.version === opts.modelVersion + deployment => extractModel(deployment)?.name === opts.model!.name ); + if (opts.model.version) { + // feature idea: smart handling of 'latest' version: treat 'latest' and the highest version number as the same + deploymentList = deploymentList.filter( + deployment => extractModel(deployment)?.version === opts.model!.version + ); + } } + if (deploymentList.length === 0) { throw new Error( 'No deployment matched the given criteria: ' + JSON.stringify(opts) @@ -61,5 +62,5 @@ export async function resolveDeployment(opts: { return deploymentList[0]; } -const modelExtractor = (deployment: AiDeployment) => +const extractModel = (deployment: AiDeployment) => deployment.details?.resources?.backend_details?.model; diff --git a/sample-code/src/aiservice.ts b/sample-code/src/aiservice.ts index 0fed80086..dd6d2468b 100644 --- a/sample-code/src/aiservice.ts +++ b/sample-code/src/aiservice.ts @@ -1,6 +1,5 @@ import { OpenAiClient, - OpenAiModels, OpenAiChatAssistantMessage } from '@sap-ai-sdk/gen-ai-hub'; @@ -12,10 +11,11 @@ const openAiClient = new OpenAiClient(); */ export async function chatCompletion(): Promise { const response = await openAiClient.chatCompletion( - OpenAiModels.GPT_35_TURBO, + 'gpt-35-turbo', { messages: [{ role: 'user', content: 'What is the capital of France?' }] - } + }, + '1234' ); const assistantMessage = response.choices[0] .message as OpenAiChatAssistantMessage; @@ -27,7 +27,7 @@ export async function chatCompletion(): Promise { * @returns An embedding vector. */ export async function computeEmbedding(): Promise { - const response = await openAiClient.embeddings(OpenAiModels.ADA_002, { + const response = await openAiClient.embeddings('text-embedding-ada-002', { input: 'Hello, world!' }); return response.data[0].embedding; From ea99958eb768bc0b49e995a5ec695b420dfe7a40 Mon Sep 17 00:00:00 2001 From: Kuhr Date: Thu, 22 Aug 2024 16:17:46 +0200 Subject: [PATCH 14/24] Linting --- .../src/client/openai/openai-client.test.ts | 2 +- .../src/client/openai/openai-client.ts | 26 ++++++++++++------- .../src/utils/deployment-resolver.test.ts | 8 +++--- .../src/utils/deployment-resolver.ts | 2 +- 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/packages/gen-ai-hub/src/client/openai/openai-client.test.ts b/packages/gen-ai-hub/src/client/openai/openai-client.test.ts index 3a07cc7e4..e11053fa2 100644 --- a/packages/gen-ai-hub/src/client/openai/openai-client.test.ts +++ b/packages/gen-ai-hub/src/client/openai/openai-client.test.ts @@ -8,7 +8,7 @@ import { OpenAiChatCompletionOutput, OpenAiChatMessage, OpenAiEmbeddingOutput, - OpenAiEmbeddingParameters, + OpenAiEmbeddingParameters } from './openai-types.js'; import { OpenAiClient } from './openai-client.js'; diff --git a/packages/gen-ai-hub/src/client/openai/openai-client.ts b/packages/gen-ai-hub/src/client/openai/openai-client.ts index 6c07f5f1c..e83bd2484 100644 --- a/packages/gen-ai-hub/src/client/openai/openai-client.ts +++ b/packages/gen-ai-hub/src/client/openai/openai-client.ts @@ -2,7 +2,6 @@ import { HttpRequestConfig } from '@sap-cloud-sdk/http-client'; import { CustomRequestConfig, executeRequest } from '@sap-ai-sdk/core'; import { DeploymentResolver, - FoundationModel, resolveDeployment } from '../../utils/deployment-resolver.js'; import { @@ -54,7 +53,9 @@ export class OpenAiClient { * @returns The completion result. */ async embeddings( - model: OpenAiEmbeddingModel | { name: OpenAiEmbeddingModel; version: string }, + model: + | OpenAiEmbeddingModel + | { name: OpenAiEmbeddingModel; version: string }, data: OpenAiEmbeddingParameters, deploymentResolver?: DeploymentResolver, requestConfig?: CustomRequestConfig @@ -80,15 +81,20 @@ export class OpenAiClient { } } -async function resolveOpenAiDeployment(model: string | { name: string; version: string }, resolver?: DeploymentResolver) { +async function resolveOpenAiDeployment( + model: string | { name: string; version: string }, + resolver?: DeploymentResolver +) { if (typeof resolver === 'string') { return resolver; } - const llm = typeof model === 'string' ? { name: model, version: 'latest' } : model; - return (await resolveDeployment({ - scenarioId: 'foundation-models', - executableId: 'azure-openai', - model: llm - }) - ).id; + const llm = + typeof model === 'string' ? { name: model, version: 'latest' } : model; + return ( + await resolveDeployment({ + scenarioId: 'foundation-models', + executableId: 'azure-openai', + model: llm + }) + ).id; } diff --git a/packages/gen-ai-hub/src/utils/deployment-resolver.test.ts b/packages/gen-ai-hub/src/utils/deployment-resolver.test.ts index 177f6b9b1..d12288451 100644 --- a/packages/gen-ai-hub/src/utils/deployment-resolver.test.ts +++ b/packages/gen-ai-hub/src/utils/deployment-resolver.test.ts @@ -28,7 +28,7 @@ describe('Deployment resolver', () => { it('should return the deployment with the correct model name', async () => { const { id, configurationId } = await resolveDeployment({ scenarioId: 'foundation-models', - model: {name: 'gpt-4o'} + model: { name: 'gpt-4o' } }); expect(id).toBe('2'); expect(configurationId).toBe('c2'); @@ -36,7 +36,7 @@ describe('Deployment resolver', () => { it('should return the deployment with the correct model name', async () => { const { id, configurationId } = await resolveDeployment({ scenarioId: 'foundation-models', - model: {name: 'gpt-4o', version: '0613'} + model: { name: 'gpt-4o', version: '0613' } }); expect(id).toBe('2'); expect(configurationId).toBe('c2'); @@ -45,7 +45,7 @@ describe('Deployment resolver', () => { await expect( resolveDeployment({ scenarioId: 'foundation-models', - model: {name: 'not existing'} + model: { name: 'not existing' } }) ).rejects.toThrow('No deployment matched the given criteria'); }); @@ -53,7 +53,7 @@ describe('Deployment resolver', () => { await expect( resolveDeployment({ scenarioId: 'foundation-models', - model: {name: 'gpt-4o', version: 'not existing'} + model: { name: 'gpt-4o', version: 'not existing' } }) ).rejects.toThrow('No deployment matched the given criteria'); }); diff --git a/packages/gen-ai-hub/src/utils/deployment-resolver.ts b/packages/gen-ai-hub/src/utils/deployment-resolver.ts index 3cc1fd776..37252d416 100644 --- a/packages/gen-ai-hub/src/utils/deployment-resolver.ts +++ b/packages/gen-ai-hub/src/utils/deployment-resolver.ts @@ -53,7 +53,7 @@ export async function resolveDeployment(opts: { ); } } - + if (deploymentList.length === 0) { throw new Error( 'No deployment matched the given criteria: ' + JSON.stringify(opts) From 7c130d749ed0f0f95324b9b1e9a56761a38c9494 Mon Sep 17 00:00:00 2001 From: Kuhr Date: Thu, 22 Aug 2024 16:21:04 +0200 Subject: [PATCH 15/24] More style --- .../src/client/openai/openai-client.ts | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/packages/gen-ai-hub/src/client/openai/openai-client.ts b/packages/gen-ai-hub/src/client/openai/openai-client.ts index e83bd2484..75a2002d2 100644 --- a/packages/gen-ai-hub/src/client/openai/openai-client.ts +++ b/packages/gen-ai-hub/src/client/openai/openai-client.ts @@ -33,14 +33,14 @@ export class OpenAiClient { deploymentResolver?: DeploymentResolver, requestConfig?: CustomRequestConfig ): Promise { - const deployment = await resolveOpenAiDeployment(model, deploymentResolver); + const deploymentId = await resolveOpenAiDeployment(model, deploymentResolver); const response = await executeRequest( { - url: `/inference/deployments/${deployment}/chat/completions`, + url: `/inference/deployments/${deploymentId}/chat/completions`, apiVersion }, data, - this.mergeRequestConfig(requestConfig) + mergeRequestConfig(requestConfig) ); return response.data; } @@ -60,25 +60,14 @@ export class OpenAiClient { deploymentResolver?: DeploymentResolver, requestConfig?: CustomRequestConfig ): Promise { - const deployment = await resolveOpenAiDeployment(model, deploymentResolver); + const deploymentId = await resolveOpenAiDeployment(model, deploymentResolver); const response = await executeRequest( - { url: `/inference/deployments/${deployment}/embeddings`, apiVersion }, + { url: `/inference/deployments/${deploymentId}/embeddings`, apiVersion }, data, - this.mergeRequestConfig(requestConfig) + mergeRequestConfig(requestConfig) ); return response.data; } - - mergeRequestConfig(requestConfig?: CustomRequestConfig): HttpRequestConfig { - return { - method: 'POST', - headers: { - 'content-type': 'application/json' - }, - params: { 'api-version': apiVersion }, - ...requestConfig - }; - } } async function resolveOpenAiDeployment( @@ -98,3 +87,14 @@ async function resolveOpenAiDeployment( }) ).id; } + +function mergeRequestConfig(requestConfig?: CustomRequestConfig): HttpRequestConfig { + return { + method: 'POST', + headers: { + 'content-type': 'application/json' + }, + params: { 'api-version': apiVersion }, + ...requestConfig + }; +} \ No newline at end of file From 0d4c7df7e69d2c05d99daa3176e0b0d45addd799 Mon Sep 17 00:00:00 2001 From: cloud-sdk-js Date: Thu, 22 Aug 2024 14:21:43 +0000 Subject: [PATCH 16/24] fix: Changes from lint --- .../src/client/openai/openai-client.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/gen-ai-hub/src/client/openai/openai-client.ts b/packages/gen-ai-hub/src/client/openai/openai-client.ts index 75a2002d2..89693e824 100644 --- a/packages/gen-ai-hub/src/client/openai/openai-client.ts +++ b/packages/gen-ai-hub/src/client/openai/openai-client.ts @@ -33,7 +33,10 @@ export class OpenAiClient { deploymentResolver?: DeploymentResolver, requestConfig?: CustomRequestConfig ): Promise { - const deploymentId = await resolveOpenAiDeployment(model, deploymentResolver); + const deploymentId = await resolveOpenAiDeployment( + model, + deploymentResolver + ); const response = await executeRequest( { url: `/inference/deployments/${deploymentId}/chat/completions`, @@ -60,7 +63,10 @@ export class OpenAiClient { deploymentResolver?: DeploymentResolver, requestConfig?: CustomRequestConfig ): Promise { - const deploymentId = await resolveOpenAiDeployment(model, deploymentResolver); + const deploymentId = await resolveOpenAiDeployment( + model, + deploymentResolver + ); const response = await executeRequest( { url: `/inference/deployments/${deploymentId}/embeddings`, apiVersion }, data, @@ -88,7 +94,9 @@ async function resolveOpenAiDeployment( ).id; } -function mergeRequestConfig(requestConfig?: CustomRequestConfig): HttpRequestConfig { +function mergeRequestConfig( + requestConfig?: CustomRequestConfig +): HttpRequestConfig { return { method: 'POST', headers: { @@ -97,4 +105,4 @@ function mergeRequestConfig(requestConfig?: CustomRequestConfig): HttpRequestCon params: { 'api-version': apiVersion }, ...requestConfig }; -} \ No newline at end of file +} From 3a5067d8542c668618088e229bcca2426912b145 Mon Sep 17 00:00:00 2001 From: Kuhr Date: Thu, 22 Aug 2024 16:31:49 +0200 Subject: [PATCH 17/24] Fix type test --- tests/type-tests/test/openai.test-d.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/type-tests/test/openai.test-d.ts b/tests/type-tests/test/openai.test-d.ts index f07528698..f6c6aa82f 100644 --- a/tests/type-tests/test/openai.test-d.ts +++ b/tests/type-tests/test/openai.test-d.ts @@ -3,7 +3,6 @@ import { OpenAiClient, OpenAiChatCompletionOutput, OpenAiEmbeddingOutput, - OpenAiModels } from '@sap-ai-sdk/gen-ai-hub'; const client = new OpenAiClient(); @@ -13,7 +12,7 @@ expectType(client); * Chat Completion. */ expectType>( - client.chatCompletion(OpenAiModels.GPT_35_TURBO, { + client.chatCompletion('gpt-4', { messages: [{ role: 'user', content: 'test prompt' }] }) ); @@ -22,11 +21,11 @@ expectType>( * Embeddings. */ expectType>( - client.embeddings(OpenAiModels.ADA_002, { + client.embeddings('text-embedding-ada-002', { input: 'test input' }) ); expectError( - client.embeddings(OpenAiModels.GPT_35_TURBO, { input: 'test input' }) + client.embeddings('gpt-35-turbo', { input: 'test input' }) ); From a7e7cf211a9d7c6964b05f066a583754e2087ee9 Mon Sep 17 00:00:00 2001 From: cloud-sdk-js Date: Thu, 22 Aug 2024 14:32:30 +0000 Subject: [PATCH 18/24] fix: Changes from lint --- tests/type-tests/test/openai.test-d.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/type-tests/test/openai.test-d.ts b/tests/type-tests/test/openai.test-d.ts index f6c6aa82f..e39c30da4 100644 --- a/tests/type-tests/test/openai.test-d.ts +++ b/tests/type-tests/test/openai.test-d.ts @@ -2,7 +2,7 @@ import { expectError, expectType } from 'tsd'; import { OpenAiClient, OpenAiChatCompletionOutput, - OpenAiEmbeddingOutput, + OpenAiEmbeddingOutput } from '@sap-ai-sdk/gen-ai-hub'; const client = new OpenAiClient(); @@ -26,6 +26,4 @@ expectType>( }) ); -expectError( - client.embeddings('gpt-35-turbo', { input: 'test input' }) -); +expectError(client.embeddings('gpt-35-turbo', { input: 'test input' })); From f16cdbd98e4b218f6bd21e8ecb3599b3e7a2c80e Mon Sep 17 00:00:00 2001 From: Kuhr Date: Thu, 22 Aug 2024 16:40:04 +0200 Subject: [PATCH 19/24] JS docs --- .../src/client/openai/openai-client.ts | 15 +++++----- .../src/client/openai/openai-types.ts | 9 ++++-- .../src/utils/deployment-resolver.ts | 28 ++++++++++++++----- 3 files changed, 34 insertions(+), 18 deletions(-) diff --git a/packages/gen-ai-hub/src/client/openai/openai-client.ts b/packages/gen-ai-hub/src/client/openai/openai-client.ts index 89693e824..443c8ad4f 100644 --- a/packages/gen-ai-hub/src/client/openai/openai-client.ts +++ b/packages/gen-ai-hub/src/client/openai/openai-client.ts @@ -16,7 +16,7 @@ import { const apiVersion = '2024-02-01'; /** - * OpenAI GPT Client. + * OpenAI Client. */ export class OpenAiClient { /** @@ -85,13 +85,12 @@ async function resolveOpenAiDeployment( } const llm = typeof model === 'string' ? { name: model, version: 'latest' } : model; - return ( - await resolveDeployment({ - scenarioId: 'foundation-models', - executableId: 'azure-openai', - model: llm - }) - ).id; + const deployment = await resolveDeployment({ + scenarioId: 'foundation-models', + executableId: 'azure-openai', + model: llm + }); + return deployment.id; } function mergeRequestConfig( diff --git a/packages/gen-ai-hub/src/client/openai/openai-types.ts b/packages/gen-ai-hub/src/client/openai/openai-types.ts index cb032b9ab..13d432f85 100644 --- a/packages/gen-ai-hub/src/client/openai/openai-types.ts +++ b/packages/gen-ai-hub/src/client/openai/openai-types.ts @@ -1,6 +1,6 @@ -// TODO: docs -/* eslint-disable */ - +/** + * Available OpenAI models for chat completion. + */ export type OpenAiChatModel = | 'gpt-4o' | 'gpt-4' @@ -9,6 +9,9 @@ export type OpenAiChatModel = | 'gpt-35-turbo-0125' | 'gpt-35-turbo-16k'; +/** + * OpenAI embedding models. + */ export type OpenAiEmbeddingModel = | 'text-embedding-ada-002' | 'text-embedding-3-small' diff --git a/packages/gen-ai-hub/src/utils/deployment-resolver.ts b/packages/gen-ai-hub/src/utils/deployment-resolver.ts index 37252d416..224c2c87c 100644 --- a/packages/gen-ai-hub/src/utils/deployment-resolver.ts +++ b/packages/gen-ai-hub/src/utils/deployment-resolver.ts @@ -4,20 +4,34 @@ import { AiDeploymentStatus } from '@sap-ai-sdk/ai-core'; -// TODO: docs -/* eslint-disable */ - +/** + * A deployment resolver can be either a deployment ID or a function that returns a full deployment object. + */ export type DeploymentResolver = DeploymentId | (() => Promise); +/** + * A deployment ID is a string that uniquely identifies a deployment. + */ export type DeploymentId = string; -export type FoundationModel = { name: string; version?: string }; +/** + * A foundation model is identifier by its name and potentially a version. + */ +export interface FoundationModel { + /** + * The name of the model. + */ + name: string; + /** + * The version of the model. + */ + version?: string; +} -// TODO: figure out what the best search criteria are /** * Query the AI Core service for a deployment that matches the given criteria. If more than one deployment matches the criteria, the first one is returned. + * @param opts - The options for the deployment resolution. * @param opts.scenarioId - The scenario ID of the deployment. * @param opts.executableId - The executable of the deployment. - * @param opts.modelName - The name of the model of the deployment. - * @param opts.modelVersion - The version of the model of the deployment. + * @param opts.model - The name and potentially version of the model to look for. * @returns An AiDeployment, if a deployment was found, fails otherwise. */ export async function resolveDeployment(opts: { From e7c5fcd4732ba20b149ead119115a36cfc9a71da Mon Sep 17 00:00:00 2001 From: Kuhr Date: Thu, 22 Aug 2024 19:15:12 +0200 Subject: [PATCH 20/24] fix leftover --- sample-code/src/aiservice.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/sample-code/src/aiservice.ts b/sample-code/src/aiservice.ts index dd6d2468b..b161682f7 100644 --- a/sample-code/src/aiservice.ts +++ b/sample-code/src/aiservice.ts @@ -10,13 +10,9 @@ const openAiClient = new OpenAiClient(); * @returns The answer from GPT. */ export async function chatCompletion(): Promise { - const response = await openAiClient.chatCompletion( - 'gpt-35-turbo', - { - messages: [{ role: 'user', content: 'What is the capital of France?' }] - }, - '1234' - ); + const response = await openAiClient.chatCompletion('gpt-35-turbo', { + messages: [{ role: 'user', content: 'What is the capital of France?' }] + }); const assistantMessage = response.choices[0] .message as OpenAiChatAssistantMessage; return assistantMessage.content!; From 6a2101c33ed97b60d0a2db26c05ee892ef6b794f Mon Sep 17 00:00:00 2001 From: Marika Marszalkowski <868536+marikaner@users.noreply.github.com> Date: Fri, 23 Aug 2024 09:33:11 +0200 Subject: [PATCH 21/24] Update packages/gen-ai-hub/src/orchestration/orchestration-client.ts --- packages/gen-ai-hub/src/orchestration/orchestration-client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gen-ai-hub/src/orchestration/orchestration-client.ts b/packages/gen-ai-hub/src/orchestration/orchestration-client.ts index c8391d081..203b10883 100644 --- a/packages/gen-ai-hub/src/orchestration/orchestration-client.ts +++ b/packages/gen-ai-hub/src/orchestration/orchestration-client.ts @@ -16,7 +16,7 @@ export class OrchestrationClient { /** * Creates a completion for the chat messages. * @param data - The input parameters for the chat completion. - * @param deploymentResolver - A deployment id or a function to retrieve it. + * @param deploymentResolver - A deployment ID or a function to retrieve it. * @param requestConfig - Request configuration. * @returns The completion result. */ From 8423be8e7e167263bf1103817ca6b14c7820487d Mon Sep 17 00:00:00 2001 From: Marika Marszalkowski <868536+marikaner@users.noreply.github.com> Date: Fri, 23 Aug 2024 10:02:06 +0200 Subject: [PATCH 22/24] Update packages/gen-ai-hub/src/utils/deployment-resolver.ts --- packages/gen-ai-hub/src/utils/deployment-resolver.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gen-ai-hub/src/utils/deployment-resolver.ts b/packages/gen-ai-hub/src/utils/deployment-resolver.ts index 224c2c87c..c59890f63 100644 --- a/packages/gen-ai-hub/src/utils/deployment-resolver.ts +++ b/packages/gen-ai-hub/src/utils/deployment-resolver.ts @@ -68,7 +68,7 @@ export async function resolveDeployment(opts: { } } - if (deploymentList.length === 0) { + if (!deploymentList.length) { throw new Error( 'No deployment matched the given criteria: ' + JSON.stringify(opts) ); From 672fe1964331eb99185cc78df29690fee57db2f5 Mon Sep 17 00:00:00 2001 From: Tom Frenken Date: Fri, 23 Aug 2024 10:14:07 +0200 Subject: [PATCH 23/24] fix typescript issue --- eslint.config.js | 2 +- packages/gen-ai-hub/src/utils/deployment-resolver.ts | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/eslint.config.js b/eslint.config.js index a53711076..308afaea7 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -8,7 +8,7 @@ export default [ rules: { 'import/namespace': 'off'} }, { - ignores: ['**/dist/**/*', '**/coverage/**/*', 'packages/ai-core/src/client/**/*'], + ignores: ['**/dist*/**/*', '**/coverage/**/*', 'packages/ai-core/src/client/**/*'], }, { files: ['**/test-util/**/*.ts', '**/packages/gen-ai-hub/src/orchestration/client/**/*'], diff --git a/packages/gen-ai-hub/src/utils/deployment-resolver.ts b/packages/gen-ai-hub/src/utils/deployment-resolver.ts index c59890f63..a2c745512 100644 --- a/packages/gen-ai-hub/src/utils/deployment-resolver.ts +++ b/packages/gen-ai-hub/src/utils/deployment-resolver.ts @@ -57,13 +57,15 @@ export async function resolveDeployment(opts: { } if (opts.model) { + const modelName = opts.model.name; deploymentList = deploymentList.filter( - deployment => extractModel(deployment)?.name === opts.model!.name + deployment => extractModel(deployment)?.name === modelName ); if (opts.model.version) { + const modelVersion = opts.model.version; // feature idea: smart handling of 'latest' version: treat 'latest' and the highest version number as the same deploymentList = deploymentList.filter( - deployment => extractModel(deployment)?.version === opts.model!.version + deployment => extractModel(deployment)?.version === modelVersion ); } } From 57579cc6dc963f18fb508843081b683a0c93e94f Mon Sep 17 00:00:00 2001 From: Tom Frenken Date: Fri, 23 Aug 2024 10:47:07 +0200 Subject: [PATCH 24/24] update unit tests --- .../orchestration-completion-post-request.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/gen-ai-hub/src/orchestration/orchestration-completion-post-request.test.ts b/packages/gen-ai-hub/src/orchestration/orchestration-completion-post-request.test.ts index eda3f7f25..919c76684 100644 --- a/packages/gen-ai-hub/src/orchestration/orchestration-completion-post-request.test.ts +++ b/packages/gen-ai-hub/src/orchestration/orchestration-completion-post-request.test.ts @@ -39,7 +39,7 @@ describe('constructCompletionPostRequest()', () => { // Todo: Adapt the test after Cloud SDK fix for: https://github.com/SAP/cloud-sdk-backlog/issues/1234 it('with model configuration and empty template', async () => { - genaihubCompletionParameters.prompt.template = []; + input.prompt.template = []; const expectedCompletionPostRequest: CompletionPostRequest = { orchestration_config: { module_configurations: { @@ -54,7 +54,7 @@ describe('constructCompletionPostRequest()', () => { } }; const completionPostRequest: CompletionPostRequest = - constructCompletionPostRequest(genaihubCompletionParameters); + constructCompletionPostRequest(input); expect(completionPostRequest).toEqual(expectedCompletionPostRequest); }); @@ -93,7 +93,7 @@ describe('constructCompletionPostRequest()', () => { }); it('with model configuration, prompt template and empty template params', async () => { - genaihubCompletionParameters.prompt = { + input.prompt = { template: [ { role: 'user', @@ -122,7 +122,7 @@ describe('constructCompletionPostRequest()', () => { input_params: {} }; const completionPostRequest: CompletionPostRequest = - constructCompletionPostRequest(genaihubCompletionParameters); + constructCompletionPostRequest(input); expect(completionPostRequest).toEqual(expectedCompletionPostRequest); });