feat: add model parameters

webisopen · Dec 5, 2024 · a7e53c9 · a7e53c9
1 parent 188c829
commit a7e53c9
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 21 deletions.
diff --git a/tests/openai/__init__.py → tests/openai-api/__init__.py b/tests/openai/__init__.py → tests/openai-api/__init__.py
diff --git a/tests/openai/example.sh → tests/openai-api/example.sh b/tests/openai/example.sh → tests/openai-api/example.sh
@@ -21,7 +21,7 @@ curl http://localhost:8000/v1/chat/completions \
     "messages": [
       {
         "role": "user",
-        "content": "Hello!"
+        "content": "btc price?"
       }
     ]
   }'
diff --git a/tests/run_test.py b/tests/run_test.py
@@ -14,16 +14,16 @@
 ]
 
 OPENSOURCE_MODELS = [
-    {"name": "qwen2", "function_call_support": True},
-    {"name": "mistral", "function_call_support": True},
-    {"name": "qwen2.5", "function_call_support": True},
-    {"name": "llama3.1", "function_call_support": True},
-    {"name": "llama3.2", "function_call_support": True},
-    {"name": "mistral-nemo", "function_call_support": True},
-    {"name": "olmo", "function_call_support": False},
-    {"name": "gemma", "function_call_support": False},
-    {"name": "llava", "function_call_support": False},
-    {"name": "deepseek-coder-v2", "function_call_support": False}
+    {"name": "qwen2", "function_call_support": True, "parameters": "7B"},
+    {"name": "mistral", "function_call_support": True, "parameters": "7B"},
+    {"name": "qwen2.5", "function_call_support": True, "parameters": "7B"},
+    {"name": "llama3.1", "function_call_support": True, "parameters": "8B"},
+    {"name": "llama3.2", "function_call_support": True, "parameters": "3B"},
+    {"name": "mistral-nemo", "function_call_support": True, "parameters": "12B"},
+    {"name": "olmo", "function_call_support": False, "parameters": "7B"},
+    {"name": "gemma", "function_call_support": False, "parameters": "7B"},
+    {"name": "llava", "function_call_support": False, "parameters": "13B"},
+    {"name": "deepseek-coder-v2", "function_call_support": False, "parameters": "16B"}
 ]
 
 
@@ -72,21 +72,22 @@ def bool_to_emoji(value):
                 'score': proprietary_results[model['name']],
                 'function_call_support': bool_to_emoji(model['function_call_support'])
             })
-    # Sort models, putting '-' scores at the end
-    proprietary_models.sort(
-        key=lambda x: float('-inf') if x['score'] == '-' else x['score'], 
-        reverse=True
-    )
-
+
     open_source_models = []
     for model in OPENSOURCE_MODELS:
         if model['name'] in opensource_results:
             open_source_models.append({
                 'name': model['name'],
                 'score': opensource_results[model['name']],
-                'function_call_support': bool_to_emoji(model['function_call_support'])
+                'function_call_support': bool_to_emoji(model['function_call_support']),
+                'parameters': model['parameters']
             })
+
     # Sort models, putting '-' scores at the end
+    proprietary_models.sort(
+        key=lambda x: float('-inf') if x['score'] == '-' else x['score'], 
+        reverse=True
+    )
     open_source_models.sort(
         key=lambda x: float('-inf') if x['score'] == '-' else x['score'], 
         reverse=True

diff --git a/tests/templates/compatible-models.mdx.j2 b/tests/templates/compatible-models.mdx.j2
@@ -30,10 +30,10 @@ Learn more:
 
 <LinkCard id="ollama" />
 
-| Name | Score (out of 100) | Function Call Support |
-|------|-------------------|---------------------|
+| Name | Parameters | Score (out of 100) | Function Call Support |
+|------|------------|-------------------|---------------------|
 {%- for model in open_source_models %}
-| {{ model.name }} | {{ model.score }} | {{ model.function_call_support }} |
+| {{ model.name }} | {{ model.parameters }} | {{ model.score }} | {{ model.function_call_support }} |
 {%- endfor %}
 
 ### Proprietary Models