From 5c05c2af3bdc8a86317772bbb437bfa2a8af620d Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Wed, 8 Nov 2023 20:06:25 -0800 Subject: [PATCH 1/4] v0 litellm --- bigcode_eval/tasks/humanevalpack_openai.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bigcode_eval/tasks/humanevalpack_openai.py b/bigcode_eval/tasks/humanevalpack_openai.py index 742382747..845581621 100644 --- a/bigcode_eval/tasks/humanevalpack_openai.py +++ b/bigcode_eval/tasks/humanevalpack_openai.py @@ -37,6 +37,7 @@ def get_prompt_base(doc, language="python"): import os import openai +import litellm import jsonlines import termcolor @@ -170,7 +171,7 @@ def __call__(self, prompt: str, n: int) -> str: ] while True: try: - response = openai.ChatCompletion.create( + response = litellm.completion( model=self._model, messages=messages, temperature=0.2, @@ -199,8 +200,8 @@ def __call__(self, prompt: str, n: int) -> str: with jsonlines.open(f"completions_{LANGUAGE}_humanevalexplaindescribe.jsonl", "r") as f: descriptions = [line["raw_generation"][0] for line in f] - openai.organization = os.getenv("OPENAI_ORGANIZATION") - openai.api_key = os.getenv("OPENAI_API_KEY") + litellm.organization = os.getenv("OPENAI_ORGANIZATION") + litellm.api_key = os.getenv("OPENAI_API_KEY") samples = [s for s in load_dataset("bigcode/humanevalpack", LANGUAGE)["test"]] From 85ced61abc2d5d70817d6c5d2b492a6e15d364d3 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Wed, 8 Nov 2023 20:07:16 -0800 Subject: [PATCH 2/4] litellm add v --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index cf80c7d24..663d33e7d 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,7 @@ "matplotlib==3.5.2", "numpy==1.21.6", "openai==0.23.0", + "litellm==0.13.2", "pandas==1.3.5", "pandas-datareader==0.10.0", "pathlib==1.0.1", From 2d74323daa77100c76a01ca526d7c8d2ede28eda Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 15 Nov 2023 12:42:59 -0800 Subject: [PATCH 3/4] Update bigcode_eval/tasks/humanevalpack_openai.py Co-authored-by: Niklas Muennighoff --- bigcode_eval/tasks/humanevalpack_openai.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bigcode_eval/tasks/humanevalpack_openai.py b/bigcode_eval/tasks/humanevalpack_openai.py index 845581621..9fa27312a 100644 --- a/bigcode_eval/tasks/humanevalpack_openai.py +++ b/bigcode_eval/tasks/humanevalpack_openai.py @@ -171,6 +171,7 @@ def __call__(self, prompt: str, n: int) -> str: ] while True: try: + # Equivalent to response = openai.ChatCompletion.create( response = litellm.completion( model=self._model, messages=messages, From 5188189cfd6185f581a9fe57de936032b5afb4ce Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 15 Nov 2023 12:43:20 -0800 Subject: [PATCH 4/4] Update bigcode_eval/tasks/humanevalpack_openai.py Co-authored-by: Niklas Muennighoff --- bigcode_eval/tasks/humanevalpack_openai.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bigcode_eval/tasks/humanevalpack_openai.py b/bigcode_eval/tasks/humanevalpack_openai.py index 9fa27312a..958740a24 100644 --- a/bigcode_eval/tasks/humanevalpack_openai.py +++ b/bigcode_eval/tasks/humanevalpack_openai.py @@ -201,6 +201,9 @@ def __call__(self, prompt: str, n: int) -> str: with jsonlines.open(f"completions_{LANGUAGE}_humanevalexplaindescribe.jsonl", "r") as f: descriptions = [line["raw_generation"][0] for line in f] + # Equivalent to: + # openai.organization = os.getenv("OPENAI_ORGANIZATION") + # openai.api_key = os.getenv("OPENAI_API_KEY") litellm.organization = os.getenv("OPENAI_ORGANIZATION") litellm.api_key = os.getenv("OPENAI_API_KEY")