Merge pull request #257 from Portkey-AI/feat/testCases

Test cases

pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools"]
 build-backend = "setuptools.build_meta"
 
 [tool.mypy]
-exclude = 'portkey_ai/_vendor'
+exclude = ['portkey_ai/_vendor', 'tests']
 ignore_missing_imports = true
 follow_imports = "silent"
 disable_error_code = ['import-untyped', 'import-not-found']
@@ -16,4 +16,4 @@ ignore_errors = true
 force-exclude = '''(portkey_ai/_vendor)/'''
 
 [tool.ruff]
-exclude =  ["portkey_ai/_vendor"]
+exclude =  ["portkey_ai/_vendor", "tests"]

tests/configs/batches/seed_tasks.jsonl

@@ -0,0 +1,21 @@
+{"id": "seed_task_1", "name": "complete_python_code", "instruction": "Suggest a completion for the following python code.", "instances": [{"input": "import json\n\ndef read_task_from_jsonl(data_file):\n    '''This function will read a .jsonl file and return the ``task`` fields in all the lines.'''", "output": "import json\n\ndef read_task_from_jsonl(data_file):\n    '''This function will read a .jsonl file and return the ``task`` fields in all the lines.'''\n    with open(data_file) as fin:\n        return [json.loads(line)['task'] for line in fin]"}], "is_classification": false}
+{"id": "seed_task_2", "name": "code_todo", "instruction": "Using the comment as a guide, replace the \"TODO\" comment in the code with the proper code.", "instances": [{"input": "def greet_to(name):\n    \"\"\"Print a greeting to the name provided.\"\"\"\n    # TODO: assert name is a string\n    print(\"Hello, \" + name + \"!\")", "output": "def greet_to(name):\n    \"\"\"Print a greeting to the name provided.\"\"\"\n    # assert name is a string\n    assert isinstance(name, str), \"name must be a string\"\n    print(\"Hello, \" + name + \"!\")"}], "is_classification": false}
+{"id": "seed_task_3", "name": "python_code_generation", "instruction": "Generate a correct python program to fulfill the following purpose:", "instances": [{"input": "Count the occurrence of an element x in a list y.", "output": "def count_x(x, y):\n    count = 0\n    for i in range(y):\n        if i == x:\n            count += 1\n    return count"}], "is_classification": false}
+{"id": "seed_task_4", "name": "code_for_bitcoin_price", "instruction": "write a code to get the current value of a Bitcoin in US dollars using the bitcoincharts API.", "instances": [{"input": "", "output": "\"\"\"\nGet the current value of a Bitcoin in US dollars using the bitcoincharts api\n\"\"\"\n\nimport requests\nimport json\n\ndef get_bitcoin_price():\n    url = 'http://api.bitcoincharts.com/v1/weighted_prices.json'\n    response = requests.get(url)\n    data = json.loads(response.text)\n    return data['USD']['24h']\n    \nif __name__ == '__main__':\n    print(get_bitcoin_price())"}], "is_classification": false}
+{"id": "seed_task_5", "name": "program_for_summation", "instruction": "Write a program to compute the sum of integers from k to n.", "instances": [{"input": "", "output": "def sum(k, n):\n    sum = 0\n    for i in range(k, n+1):\n        sum += i\n    return sum"}], "is_classification": false}
+{"id": "seed_task_6", "name": "javascript_date_and_time", "instruction": "Write a script in JavaScript for printing the current date and time.", "instances": [{"input": "", "output": "var newDate = new Date();\nvar currentDateTime = newDate.toLocaleString();\nconsole.log(currentDateTime);"}], "is_classification": false}
+{"id": "seed_task_7", "name": "python_leetcode", "instruction": "Write Python code to solve this leetcode problem.", "instances": [{"input": "Problem: You are given two non-empty linked lists representing two non-negative integers. The digits are stored in reverse order, and each of their nodes contains a single digit. Add the two numbers and return the sum as a linked list.\n\nYou may assume the two numbers do not contain any leading zero, except the number 0 itself.", "output": "class Solution(object):\n    def addTwoNumbers(self, l1, l2):\n        \"\"\"\n        :type l1: ListNode\n        :type l2: ListNode\n        :rtype: ListNode\n        \"\"\"\n        carry = 0\n        root = n = ListNode(0)\n        while l1 or l2 or carry:\n            v1 = v2 = 0\n            if l1:\n                v1 = l1.val\n                l1 = l1.next\n            if l2:\n                v2 = l2.val\n                l2 = l2.next\n            carry, val = divmod(v1+v2+carry, 10)\n            n.next = ListNode(val)\n            n = n.next\n        return root.next"}], "is_classification": false}
+{"id": "seed_task_8", "name": "pos_tag_for_word", "instruction": "Identify the pos tag of the word in the given sentence.", "instances": [{"input": "Word: excited\nSentence: She felt so excited that her application got approval.", "output": "Adjective, JJ."}], "is_classification": true}
+{"id": "seed_task_9", "name": "code_syntax_error", "instruction": "You need to read a code and detect if there is a syntax error or not. Outpus true if there is error, output false if there is not.", "instances": [{"input": "def quick_sort(arr):\n    if len(arr) < 2:\n        return arr\n    else:\n        pivot = arr[0]\n        less = [i for i in arr[1:] if i <= pivot]\n        greater = [i for i in arr[1:] if i > pivot]\n        return quick_sort(less) + [pivot] + quick_sort(greater)", "output": "false"}], "is_classification": true}
+{"id": "seed_task_10", "name": "find_prime_numbers", "instruction": "Write a Python function to find all prime numbers up to the given number n.", "instances": [{"input": "", "output": "def is_prime(n):\n if n <= 1:\n return False\n for i in range(2, int(n**0.5) + 1):\n if n % i == 0:\n return False\n return True\n\ndef find_prime_numbers(n):\n primes = []\n for i in range(2, n + 1):\n if is_prime(i):\n primes.append(i)\n return primes"}], "is_classification": false}
+{"id": "seed_task_11", "name": "csharp_hello_world", "instruction": "Write a simple C# program that prints \"Hello, World!\".", "instances": [{"input": "", "output": "using System;\n\nnamespace HelloWorld\n{\n class Program\n {\n static void Main(string[] args)\n {\n Console.WriteLine(\"Hello, World!\");\n }\n }\n}"}], "is_classification": false}
+{"id": "seed_task_12", "name": "sql_query", "instruction": "Write an SQL query to select all columns from a table named 'employees' where the age is greater than 30.", "instances": [{"input": "", "output": "SELECT * FROM employees WHERE age > 30;"}], "is_classification": false}
+{"id": "seed_task_13", "name": "palindrome_check", "instruction": "Write a Python function to check if a given word is a palindrome or not. A palindrome is a word that reads the same backwards as forwards.", "instances": [{"input": "", "output": "def is_palindrome(word):\n return word == word[::-1]"}], "is_classification": false}
+{"id": "seed_task_14", "name": "javascript_alert", "instruction": "Write a JavaScript code snippet to show an alert message with the text \"Welcome to my website!\".", "instances": [{"input": "", "output": "alert('Welcome to my website!');"}], "is_classification": false}
+{"id": "seed_task_15", "name": "python_reverse_list", "instruction": "Write a Python function to reverse a given list of integers.", "instances": [{"input": "", "output": "def reverse_list(numbers):\n return numbers[::-1]"}], "is_classification": false}
+{"id": "seed_task_16", "name": "html_basic_structure", "instruction": "Write a basic HTML structure with a title \"My First Web Page\".", "instances": [{"input": "", "output": "<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n <meta charset=\"UTF-8\">\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">\n <title>My First Web Page</title>\n</head>\n<body>\n\n</body>\n</html>"}], "is_classification": false}
+{"id": "seed_task_17", "name": "c_plus_plus_hello_world", "instruction": "Write a simple C++ program that prints \"Hello, World!\".", "instances": [{"input": "", "output": "#include <iostream>\n\nint main() {\n std::cout << \"Hello, World!\" << std::endl;\n return 0;\n}"}], "is_classification": false}
+{"id": "seed_task_18", "name": "python_summation", "instruction": "Write a Python function that takes two numbers as input and returns their sum.", "instances": [{"input": "5, 7", "output": "def add_numbers(a, b):\n    return a + b\n\nresult = add_numbers(5, 7)\nprint(result)"}], "is_classification": false}
+{"id": "seed_task_19", "name": "java_loop_array", "instruction": "Create a JavaScript function that takes an array of numbers and returns the product of all elements.", "instances": [{"input": "[2, 3, 4]", "output": "function multiplyArrayElements(arr) {\n    let product = 1;\n    for (const num of arr) {\n        product *= num;\n    }\n    return product;\n}\n\nconst result = multiplyArrayElements([2, 3, 4]);\nconsole.log(result);"}], "is_classification": false}
+{"id": "seed_task_20", "name": "shell_total_lines_in_txt_file", "instruction": "Can you provide a shell script that displays the total number of lines in all '*.txt' files in the current directory?", "instances": [{"input": "\"file1.txt:\\nHello\\nWorld\\n\\nfile2.txt:\\nAnother\\nExample\"", "output": "#!/bin/sh\n\ntotal_lines=0\n\nfor file in *.txt; do\n    lines=$(wc -l < \"$file\")\n    total_lines=$((total_lines + lines))\ndone\n\necho \"Total lines: $total_lines\""}], "is_classification": false}
+{"id": "seed_task_21", "name": "java_reverse_string_no_builtin", "instruction": "Implement a Java method to reverse a string without using any built-in functions.", "instances": [{"input": "\"Hello, World!\"", "output": "public class StringReversal {\n    public static String reverseString(String input) {\n        char[] charArray = input.toCharArray();\n        int left = 0;\n        int right = charArray.length - 1;\n\n        while (left < right) {\n            char temp = charArray[left];\n            charArray[left] = charArray[right];\n            charArray[right] = temp;\n\n            left++;\n            right--;\n        }\n        return new String(charArray);\n    }\n\n    public static void main(String[] args) {\n        String str = \"Hello, World!\";\n        System.out.println(\"Reversed string: \" + reverseString(str));\n    }\n}"}], "is_classification": false}

tests/manual_test_async_feedback.py

@@ -0,0 +1,32 @@
+import asyncio
+from portkey_ai import AsyncPortkey
+import os
+
+
+portkey = AsyncPortkey(
+    api_key=os.environ.get("PORTKEY_API_KEY"),
+)
+
+
+traceId = "0c41ce35-b321-4484-bead-1c21eae02996"
+
+
+async def main():
+    print("Step: Create Feedback")
+    result = await portkey.feedback.create(
+        trace_id=traceId,
+        value="1",
+    )
+    print(result)
+
+    update_feedback_id = result.feedback_ids[0]
+
+    print("Step: Update Feedback")
+    result = await portkey.feedback.update(
+        feedback_id=update_feedback_id,
+        value="7",
+    )
+    print(result)
+
+
+asyncio.run(main())

tests/manual_test_async_thread_query.py

@@ -0,0 +1,70 @@
+import asyncio
+from portkey_ai import AsyncPortkey, AsyncAssistants, AsyncMessages, AsyncUploads
+import os
+
+
+portkey = AsyncPortkey(
+    api_key=os.environ.get("PORTKEY_API_KEY"),
+    virtual_key=os.environ.get("OPENAI_VIRTUAL_KEY"),
+)
+
+file_to_upload_path = "./tests/configs/threads/sample.pdf"
+
+
+async def main():
+    print("Step 1: Create Assistant")
+    assistant = await AsyncAssistants(portkey).create(
+        model="gpt-4o-mini",
+        description="for testing purposes",
+        tools=[{"type": "file_search"}],
+    )
+    print(assistant)
+
+    print("Step 2: Create Upload")
+    upload = await AsyncUploads(portkey).create(
+        purpose="assistants",
+        bytes=os.stat(file_to_upload_path).st_size,  # get total bytes
+        filename=file_to_upload_path.split("/")[-1],
+        mime_type="application/pdf",
+    )
+    print(upload)
+
+    print("Step 3: Create one or more parts for the upload_id received from step 1")
+    part = await AsyncUploads(portkey).parts.create(
+        data=open(file_to_upload_path, "rb").read(), upload_id=upload.id
+    )
+    print(part)
+
+    print("Step 4: Complete the upload")
+    complete_upload = await AsyncUploads(portkey).complete(
+        upload_id=upload.id, part_ids=[part.id]
+    )
+    print(complete_upload)
+
+    print("Step 5: Create a run and poll")
+    run = await portkey.beta.threads.create_and_run_poll(
+        thread={
+            "messages": [
+                {
+                    "role": "user",
+                    "content": "What is this document about?",
+                    # Attach the new file to the message.
+                    "attachments": [
+                        {
+                            "file_id": complete_upload.file.id,
+                            "tools": [{"type": "file_search"}],
+                        }
+                    ],
+                }
+            ]
+        },
+        assistant_id=assistant.id,
+    )
+    print(run)
+
+    print("Step 6: Get the list of the messages")
+    messages = await AsyncMessages(portkey).list(thread_id=run.thread_id)
+    print(messages)
+
+
+asyncio.run(main())

tests/manual_test_batches.py

@@ -0,0 +1,33 @@
+import os
+from portkey_ai import Portkey
+
+portkey = Portkey(
+    api_key="Xgv7++HhSeF70UzdVSsmtJuGYe0=", virtual_key="portkey-welcome-e248d3"
+)
+
+
+file_name = "./tests/configs/batches/seed_tasks.jsonl"
+
+
+print("Step 1: Create a batch file")
+batch_file = portkey.files.create(file=open(file_name, "rb"), purpose="batch")
+print(batch_file)
+
+
+print("Step 2: Create a batch job")
+batch_job = portkey.batches.create(
+    input_file_id=batch_file.id,
+    endpoint="/v1/chat/completions",
+    completion_window="24h",
+)
+print(batch_job)
+
+print("Step 3: Retrieve the batch job")
+batch_job = portkey.batches.retrieve(batch_job.id)
+print(batch_job)
+
+
+print("Step 4: Retrieve the result")
+result_file_id = batch_job.output_file_id
+result = portkey.files.content(result_file_id).content
+print(result)

tests/manual_test_feedback.py

@@ -0,0 +1,26 @@
+import os
+from portkey_ai import Portkey
+
+
+portkey = Portkey(
+    api_key=os.environ.get("PORTKEY_API_KEY"),
+)
+
+
+traceId = "<trace_id>"
+
+
+print("Step: Create Feedback")
+result = portkey.feedback.create(
+    trace_id=traceId,
+    value="1",
+)
+print(result)
+
+
+print("Step: Update Feedback")
+result = portkey.feedback.update(
+    feedback_id=traceId,
+    value="7",
+)
+print(result)

tests/manual_test_threads_query.py

@@ -0,0 +1,64 @@
+import os
+from portkey_ai import Assistants, Messages, Portkey, Uploads
+
+# Initialize the Portkey client
+portkey = Portkey(
+    api_key=os.environ.get("PORTKEY_API_KEY"),
+    virtual_key=os.environ.get("OPENAI_VIRTUAL_KEY"),
+)
+
+file_to_upload_path = "./tests/configs/threads/sample.pdf"
+
+
+def test_upload_file_and_query() -> None:
+    print("Step 1: Create Assistant")
+    assistant = Assistants(portkey).create(
+        model="gpt-4o-mini",
+        description="for testing purposes",
+        tools=[{"type": "file_search"}],
+    )
+    print(assistant)
+
+    print("Step 2: Create Upload")
+    upload = Uploads(portkey).create(
+        purpose="assistants",
+        bytes=os.stat(file_to_upload_path).st_size,  # get total bytes
+        filename=file_to_upload_path.split("/")[-1],
+        mime_type="application/pdf",
+    )
+    print(upload)
+
+    print("Step 3: Create one or more parts for the upload_id received from step 1")
+    part = Uploads(portkey).parts.create(
+        data=open(file_to_upload_path, "rb").read(), upload_id=upload.id
+    )
+    print(part)
+
+    print("Step 4: Complete the upload")
+    complete_upload = Uploads(portkey).complete(upload_id=upload.id, part_ids=[part.id])
+    print(complete_upload)
+
+    print("Step 5: Create a run and poll")
+    run = portkey.beta.threads.create_and_run_poll(
+        thread={
+            "messages": [
+                {
+                    "role": "user",
+                    "content": "What is this document about?",
+                    # Attach the new file to the message.
+                    "attachments": [
+                        {
+                            "file_id": complete_upload.file.id,
+                            "tools": [{"type": "file_search"}],
+                        }
+                    ],
+                }
+            ]
+        },
+        assistant_id=assistant.id,
+    )
+    print(run)
+
+    print("Step 6: Get the list of the messages")
+    messages = Messages(portkey).list(thread_id=run.thread_id)
+    print(messages)