Merge pull request #26 from OutpostHQ/feat/inference

added inference endpoint

.changeset/hot-vans-compete.md

@@ -0,0 +1,5 @@
+---
+'outpostkit': patch
+---
+
+Inference Endpoints

src/comet.ts

@@ -10,7 +10,7 @@ import type {
   TCometPromptResponse,
   ICometSession,
   PromptOptions,
-} from './types';
+} from './types/comet';
 import {
   streamPromptWithAxios,
   streamPromptWithEventStreaming,

src/helpers.ts

@@ -156,6 +156,8 @@ export const streamPromptWithEventStreaming = async (
           } catch (e) {
             throw new ClientError('Encountered error while parsing response into JSON.');
           }
+        } else if (msg.event==='error') {
+
         }
       },
       // onclose() {

src/inference.ts

@@ -0,0 +1,92 @@
+import axios, { type AxiosInstance } from 'axios';
+import { API_V1_URL } from './constants';
+// import {
+//   streamPromptWithAxios,
+//   streamPromptWithEventStreaming,
+//   streamPromptWithNativeFetch,
+// } from 'helpers';
+import { IInference, PromptPayload, PromptOptions } from 'types/inference';
+
+export class Inference implements IInference {
+  readonly apiKey: string;
+  readonly inferenceId: string;
+  private readonly inferenceAPI: AxiosInstance;
+
+  constructor(apiKey: string, inferenceId: string) {
+    this.apiKey = apiKey;
+    this.inferenceId = inferenceId;
+
+    this.inferenceAPI = axios.create({
+      baseURL: `${API_V1_URL}/inference/${inferenceId}`,
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+        'Content-Type': 'application/json',
+      },
+    });
+  }
+
+  async getInferenceInfo() {
+    const { data } = await this.inferenceAPI.get('/');
+    return data;
+  }
+
+  async prompt(
+    payload: PromptPayload,
+    handleNewText?: (data: string) => void | Promise<void>,
+    options?: PromptOptions
+  ) {
+    return { tbd: true };
+    //TODO: better error handling
+    // if (payload.stream) {
+    //   if (typeof window !== 'undefined') {
+    //     if (options?.useNativeFetch) {
+    //       const resp = await streamPromptWithNativeFetch(
+    //         this.cometId,
+    //         this.apiKey,
+    //         payload,
+    //         handleNewText
+    //       );
+    //       // @ts-ignore
+    //       if (resp.error) {
+    //         // @ts-ignore
+    //         throw new Error(resp.error);
+    //       } else {
+    //         return resp as TCometPromptResponse;
+    //       }
+    //     } else {
+    //       const resp = await streamPromptWithEventStreaming(
+    //         this.inferenceAPI,
+    //         this.apiKey,
+    //         payload,
+    //         handleNewText
+    //       );
+    //       // @ts-ignore
+    //       if (resp.error) {
+    //         // @ts-ignore
+    //         throw new Error(resp.error);
+    //       } else {
+    //         return resp as TCometPromptResponse;
+    //       }
+    //     }
+    //   } else {
+    //     const resp = await streamPromptWithAxios(this.inferenceAPI, payload, handleNewText);
+    //     // @ts-ignore
+    //     if (resp.error) {
+    //       // @ts-ignore
+    //       throw new Error(resp.error);
+    //     } else {
+    //       return resp as TCometPromptResponse;
+    //     }
+    //   }
+    // } else {
+    //   const { data } = await this.cometAPI.post(`/prompt`, payload);
+    //   return data as TCometPromptResponse;
+    // }
+  }
+
+  async delete(): Promise<void> {
+    await this.inferenceAPI.delete('/');
+  }
+}
+
+export default Inference;

src/types.ts

@@ -50,7 +50,6 @@ export interface IComet {
     ICometConversation & { messages: ICometMessage[]; stats: ICometConversationStats | null }
   >;
 }
-
 export interface IndexInput {
   indexId?: string;
   id?: string;

src/types/comet.ts

@@ -0,0 +1,176 @@
+// Comet Client
+export interface IComet {
+  prompt: (
+    payload: PromptPayload,
+    handleNewText?: (data: string) => void | Promise<void>,
+    options?: PromptOptions
+  ) => Promise<TCometPromptResponse>;
+  update: (payload: UpdateCometPayload) => Promise<void>;
+  updateModel: (payload: UpdateModelPayload) => Promise<void>;
+  getMessage: (payload: GetMessagePayload) => Promise<ICometMessage>;
+  takeConversationFeedback: (payload: ProvideMessageFeedbackPayload) => Promise<void>;
+  deleteComet: () => Promise<void>;
+  getCometInfo: () => Promise<ICometInfo>;
+  listSessions: (payload: ListSessionsPayload) => Promise<ICometSession[]>;
+  getSession: (payload: GetSessionPayload) => Promise<ICometSession>;
+  listConversations: <M extends boolean, S extends boolean>(
+    payload: ListConversationsPayload
+  ) => Promise<
+    Array<
+      ICometConversation & {
+        messages: M extends true ? ICometMessage[] : never;
+        stats: S extends true ? ICometConversationStats | null : never;
+      }
+    >
+  >;
+  getConversation: (
+    payload: GetConversationPayload
+  ) => Promise<
+    ICometConversation & { messages: ICometMessage[]; stats: ICometConversationStats | null }
+  >;
+}
+
+export interface PromptOptions {
+  useNativeFetch?: boolean;
+}
+
+export interface PromptPayload {
+  input: string;
+  prompt_variables?: Record<string, string>;
+  channel?: string;
+  visitorId?: string;
+  sessionId?: string;
+  stream: boolean;
+  configs?: {
+    max_tokens?: number;
+    temperature?: number;
+    top_p?: number;
+    presence_penalty?: number;
+    frequency_penalty?: number;
+  };
+}
+
+export interface ListConversationsPayload {
+  sessionId: string;
+  messages?: boolean;
+  stats?: boolean;
+}
+
+export interface ListSessionsPayload {
+  userId?: string;
+  channel?: string;
+  visitorId?: string;
+}
+export interface GetSessionPayload {
+  sessionId: string;
+}
+export interface GetConversationPayload {
+  conversationId: string;
+}
+export interface GetMessagePayload {
+  messageId: string;
+}
+
+export interface ProvideMessageFeedbackPayload {
+  conversationId: string;
+  vote?: boolean;
+  feedback?: string;
+  meta?: object;
+}
+
+export interface UpdateCometPayload {
+  sectionsMatchThreshold?: number;
+  sectionMatchCount?: number;
+  name?: string;
+}
+export type TCometModelType = 'selfhosted' | 'thirdparty';
+
+export interface UpdateModelPayload {
+  type?: TCometModelType;
+  details?: { name: string; vendor: 'openai' };
+  configs?: object;
+}
+
+export type CometAIModelType = 'text' | 'chat';
+
+export interface ICometInfo {
+  projectId: string;
+  id: string;
+  createdAt: string;
+  updatedAt: string;
+  creatorId: string;
+  thirdPartyKeyId: string | null;
+  confluxId: string | null;
+  name: string;
+  configs: Record<string, any> | null;
+  whitelistedDomains: string[];
+  promptVariables: Record<string, string>;
+  promptTemplate: string | null;
+  promptTokenLimit: number | null;
+  sectionsMatchThreshold: number | null;
+  sectionMatchCount: number | null;
+  contextTokenCutoff: number | null;
+  model: string;
+  modelVendor: string;
+  modelType: CometAIModelType;
+  conversationHistoryCutoff?: string;
+}
+
+export interface ICometSession {
+  id: string;
+  channel: string;
+  metadata: Record<string, any>;
+  userId: string;
+  visitorId: string | null;
+  createdAt: string;
+  updatedAt: string;
+}
+
+export interface ICometConversationStats {
+  id: string;
+  feedback: string | null;
+  noResponse: boolean;
+  upvoted: boolean;
+  updatedAt: string;
+  downvoted: boolean;
+  processed: boolean;
+}
+
+export interface ICometConversation {
+  id: string;
+  metadata: Record<string, any> | null;
+  createdAt: string;
+  updatedAt: string;
+}
+
+export type CometMessageAuthor = 'agent' | 'human' | 'system' | 'function';
+
+export interface ICometMessage {
+  id: string;
+  text: string;
+  from: CometMessageAuthor;
+  meta: Record<string, any> | null;
+  conversationId: string;
+  createdAt: string;
+}
+
+export type TCometPromptResponse = {
+  generations: string[];
+  meta: {
+    referencePaths?: string[];
+    referencesWithSources?: {
+      path: string;
+      source_id: string;
+    }[];
+  };
+  usage: {
+    prompt_tokens: number;
+    completion_tokens: number;
+  };
+  conversationId?: string;
+  sessionId?: string;
+};
+
+export type TCometPromptStreamResponseError = {
+  error: string;
+};

src/types/inference.ts

@@ -0,0 +1,17 @@
+export interface IInference {
+  prompt: (
+    payload: PromptPayload & Record<string, string>,
+    handleNewText?: (data: string) => void | Promise<void>,
+    options?: PromptOptions
+  ) => Promise<any>;
+  getInferenceInfo: () => Promise<any>;
+}
+
+export interface PromptOptions {
+  useNativeFetch?: boolean;
+}
+
+export interface PromptPayload {
+  prompt: string;
+  stream: boolean;
+}

src/utils/inference/io.ts

@@ -0,0 +1,71 @@
+export const VLLMPromptConfigs = {
+  type: 'object',
+  properties: {
+    n: {
+      type: 'integer',
+      minimum: 1,
+    },
+    best_of: {
+      type: ['integer', 'null'],
+    },
+    presence_penalty: {
+      type: 'number',
+      minimum: -2.0,
+      maximum: 2.0,
+    },
+    frequency_penalty: {
+      type: 'number',
+      minimum: -2.0,
+      maximum: 2.0,
+    },
+    temperature: {
+      type: 'number',
+      minimum: 0.0,
+    },
+    top_p: {
+      type: 'number',
+      minimum: 0.0,
+      maximum: 1.0,
+    },
+    top_k: {
+      type: 'integer',
+      minimum: -1,
+    },
+    use_beam_search: {
+      type: 'boolean',
+    },
+    length_penalty: {
+      type: 'number',
+      minimum: 0.0,
+    },
+    early_stopping: {
+      type: ['boolean', 'string'],
+    },
+    stop: {
+      type: ['array', 'string', 'null'],
+      items: {
+        type: 'string',
+      },
+    },
+    stop_token_ids: {
+      type: 'array',
+      items: {
+        type: 'integer',
+      },
+    },
+    ignore_eos: {
+      type: 'boolean',
+    },
+    max_tokens: {
+      type: 'integer',
+      minimum: 1,
+    },
+    logprobs: {
+      type: ['integer', 'null'],
+      minimum: 0,
+    },
+    skip_special_tokens: {
+      type: 'boolean',
+    },
+  },
+};