Add openai/gpt-3.5-turbo-instruct model (#2376)

stanford-crfm · Feb 20, 2024 · 0b68983 · 0b68983
1 parent 149141c
commit 0b68983
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 1 deletion.
diff --git a/src/helm/config/model_deployments.yaml b/src/helm/config/model_deployments.yaml
@@ -1050,6 +1050,14 @@ model_deployments:
   ## GPT 3.5 Turbo Models
   # ChatGPT: https://openai.com/blog/chatgpt
 
+  - name: openai/gpt-3.5-turbo-instruct
+    model_name: openai/gpt-3.5-turbo-instruct
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 4096
+    max_sequence_length: 4097
+    client_spec:
+      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
+
   # The claimed sequence length is 4096, but as of 2023-03-07, the empirical usable
   # sequence length is smaller at 4087 with one user input message and one assistant
   # output message because ChatGPT uses special tokens for message roles and boundaries.

diff --git a/src/helm/config/model_metadata.yaml b/src/helm/config/model_metadata.yaml
@@ -1389,6 +1389,14 @@ models:
   ## GPT 3.5 Turbo Models
   # ChatGPT: https://openai.com/blog/chatgpt
 
+  - name: openai/gpt-3.5-turbo-instruct
+    display_name: GPT-3.5 Turbo Instruct
+    description: Similar capabilities as GPT-3 era models. Compatible with legacy Completions endpoint and not Chat Completions.
+    creator_organization_name: OpenAI
+    access: limited
+    release_date: 2023-09-18
+    tags: [TEXT_MODEL_TAG, OPENAI_CHATGPT_MODEL_TAG, LIMITED_FUNCTIONALITY_TEXT_MODEL_TAG, INSTRUCTION_FOLLOWING_MODEL_TAG]
+
   - name: openai/gpt-3.5-turbo-0301
     display_name: GPT-3.5 Turbo (0301)
     description: Sibling model of text-davinci-003 that is optimized for chat but works well for traditional completions tasks as well. Snapshot from 2023-03-01.

diff --git a/src/helm/proxy/clients/openai_client.py b/src/helm/proxy/clients/openai_client.py
@@ -49,7 +49,11 @@ def __init__(
         self.client = OpenAI(api_key=api_key, organization=org_id, base_url=base_url)
 
     def _is_chat_model_engine(self, model_engine: str):
-        return model_engine.startswith("gpt-3.5") or model_engine.startswith("gpt-4")
+        if model_engine == "gpt-3.5-turbo-instruct":
+            return False
+        elif model_engine.startswith("gpt-3.5") or model_engine.startswith("gpt-4"):
+            return True
+        return False
 
     def _get_cache_key(self, raw_request: Dict, request: Request):
         cache_key = CachingClient.make_cache_key(raw_request, request)