From a7e53c92ce6af5a946e7395070318e04cfb2bd1e Mon Sep 17 00:00:00 2001
From: Thomas <wxy_000000@qq.com>
Date: Thu, 5 Dec 2024 09:36:37 +0800
Subject: [PATCH] feat: add model parameters

---
 tests/{openai => openai-api}/__init__.py |  0
 tests/{openai => openai-api}/example.sh  |  2 +-
 tests/run_test.py                        | 35 ++++++++++++------------
 tests/templates/compatible-models.mdx.j2 |  6 ++--
 4 files changed, 22 insertions(+), 21 deletions(-)
 rename tests/{openai => openai-api}/__init__.py (100%)
 rename tests/{openai => openai-api}/example.sh (93%)

diff --git a/tests/openai/__init__.py b/tests/openai-api/__init__.py
similarity index 100%
rename from tests/openai/__init__.py
rename to tests/openai-api/__init__.py
diff --git a/tests/openai/example.sh b/tests/openai-api/example.sh
similarity index 93%
rename from tests/openai/example.sh
rename to tests/openai-api/example.sh
index fbb304e2..0c03447c 100644
--- a/tests/openai/example.sh
+++ b/tests/openai-api/example.sh
@@ -21,7 +21,7 @@ curl http://localhost:8000/v1/chat/completions \
     "messages": [
       {
         "role": "user",
-        "content": "Hello!"
+        "content": "btc price?"
       }
     ]
   }'
diff --git a/tests/run_test.py b/tests/run_test.py
index aca1b19e..67e96567 100644
--- a/tests/run_test.py
+++ b/tests/run_test.py
@@ -14,16 +14,16 @@
 ]
 
 OPENSOURCE_MODELS = [
-    {"name": "qwen2", "function_call_support": True},
-    {"name": "mistral", "function_call_support": True},
-    {"name": "qwen2.5", "function_call_support": True},
-    {"name": "llama3.1", "function_call_support": True},
-    {"name": "llama3.2", "function_call_support": True},
-    {"name": "mistral-nemo", "function_call_support": True},
-    {"name": "olmo", "function_call_support": False},
-    {"name": "gemma", "function_call_support": False},
-    {"name": "llava", "function_call_support": False},
-    {"name": "deepseek-coder-v2", "function_call_support": False}
+    {"name": "qwen2", "function_call_support": True, "parameters": "7B"},
+    {"name": "mistral", "function_call_support": True, "parameters": "7B"},
+    {"name": "qwen2.5", "function_call_support": True, "parameters": "7B"},
+    {"name": "llama3.1", "function_call_support": True, "parameters": "8B"},
+    {"name": "llama3.2", "function_call_support": True, "parameters": "3B"},
+    {"name": "mistral-nemo", "function_call_support": True, "parameters": "12B"},
+    {"name": "olmo", "function_call_support": False, "parameters": "7B"},
+    {"name": "gemma", "function_call_support": False, "parameters": "7B"},
+    {"name": "llava", "function_call_support": False, "parameters": "13B"},
+    {"name": "deepseek-coder-v2", "function_call_support": False, "parameters": "16B"}
 ]
 
 
@@ -72,21 +72,22 @@ def bool_to_emoji(value):
                 'score': proprietary_results[model['name']],
                 'function_call_support': bool_to_emoji(model['function_call_support'])
             })
-    # Sort models, putting '-' scores at the end
-    proprietary_models.sort(
-        key=lambda x: float('-inf') if x['score'] == '-' else x['score'], 
-        reverse=True
-    )
-
+    
     open_source_models = []
     for model in OPENSOURCE_MODELS:
         if model['name'] in opensource_results:
             open_source_models.append({
                 'name': model['name'],
                 'score': opensource_results[model['name']],
-                'function_call_support': bool_to_emoji(model['function_call_support'])
+                'function_call_support': bool_to_emoji(model['function_call_support']),
+                'parameters': model['parameters']
             })
+    
     # Sort models, putting '-' scores at the end
+    proprietary_models.sort(
+        key=lambda x: float('-inf') if x['score'] == '-' else x['score'], 
+        reverse=True
+    )
     open_source_models.sort(
         key=lambda x: float('-inf') if x['score'] == '-' else x['score'], 
         reverse=True
diff --git a/tests/templates/compatible-models.mdx.j2 b/tests/templates/compatible-models.mdx.j2
index 5bc8f70f..be2be2d9 100644
--- a/tests/templates/compatible-models.mdx.j2
+++ b/tests/templates/compatible-models.mdx.j2
@@ -30,10 +30,10 @@ Learn more:
 
 <LinkCard id="ollama" />
 
-| Name | Score (out of 100) | Function Call Support |
-|------|-------------------|---------------------|
+| Name | Parameters | Score (out of 100) | Function Call Support |
+|------|------------|-------------------|---------------------|
 {%- for model in open_source_models %}
-| {{ model.name }} | {{ model.score }} | {{ model.function_call_support }} |
+| {{ model.name }} | {{ model.parameters }} | {{ model.score }} | {{ model.function_call_support }} |
 {%- endfor %}
 
 ### Proprietary Models