From ac901f2bb6d2a75cc33f2b6871be076e548e94d0 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Thu, 9 Nov 2023 17:16:32 +0900 Subject: [PATCH] Change default Llama2 prompt to Japananese Currently the default Llama2 prompt is the default English one. However, since all the tasks are in Japanese, that doesn't make a lot of sense, and can cause the model to answer in English. This makes the default prompt Japanese and leaves the English prompt commented out for reference. --- lm_eval/tasks/ja/jaqket_v2.py | 4 +++- lm_eval/tasks/ja/jcola.py | 3 ++- lm_eval/tasks/ja/jcommonsenseqa.py | 3 ++- lm_eval/tasks/ja/jnli.py | 3 ++- lm_eval/tasks/ja/jsquad.py | 3 ++- lm_eval/tasks/ja/marc_ja.py | 3 ++- lm_eval/tasks/ja/mgsm.py | 4 +++- lm_eval/tasks/ja/wikilingua.py | 3 ++- lm_eval/tasks/ja/xlsum_ja.py | 3 ++- 9 files changed, 20 insertions(+), 9 deletions(-) diff --git a/lm_eval/tasks/ja/jaqket_v2.py b/lm_eval/tasks/ja/jaqket_v2.py index b6ba389003..3a25183e4d 100644 --- a/lm_eval/tasks/ja/jaqket_v2.py +++ b/lm_eval/tasks/ja/jaqket_v2.py @@ -376,7 +376,9 @@ class JAQKETV2WithLlama2(JAQKETV2WithJAAlpacaPrompt): """ PROMPT_VERSION = 0.6 - DEFAULT_SYSTEM_PROMPT = """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""" + # This is the English prompt. + # DEFAULT_SYSTEM_PROMPT = """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""" + DEFAULT_SYSTEM_PROMPT = "あなたは役立つアシスタントです。" SYSTEM_PROMPT = os.getenv("SYSTEM_PROMPT", DEFAULT_SYSTEM_PROMPT) DESCRIPTION = f"[INST] <>\n{SYSTEM_PROMPT}\n<>\n\n" FEWSHOT_SEP = " [INST] " diff --git a/lm_eval/tasks/ja/jcola.py b/lm_eval/tasks/ja/jcola.py index 9189a0f3f7..0cf9db23fb 100644 --- a/lm_eval/tasks/ja/jcola.py +++ b/lm_eval/tasks/ja/jcola.py @@ -139,7 +139,8 @@ class JCoLAWithLlama2(JCoLAWithJAAlpacaPrompt): """ PROMPT_VERSION = 0.6 - DEFAULT_SYSTEM_PROMPT = """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""" + # DEFAULT_SYSTEM_PROMPT = """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""" + DEFAULT_SYSTEM_PROMPT = "あなたは役立つアシスタントです。" SYSTEM_PROMPT = os.getenv("SYSTEM_PROMPT", DEFAULT_SYSTEM_PROMPT) DESCRIPTION = f"[INST] <>\n{SYSTEM_PROMPT}\n<>\n\n" FEWSHOT_SEP = " [INST] " diff --git a/lm_eval/tasks/ja/jcommonsenseqa.py b/lm_eval/tasks/ja/jcommonsenseqa.py index c7d94dc95a..bd7036d591 100644 --- a/lm_eval/tasks/ja/jcommonsenseqa.py +++ b/lm_eval/tasks/ja/jcommonsenseqa.py @@ -251,7 +251,8 @@ class JCommonsenseQAWithLlama2(JCommonsenseQA): """ PROMPT_VERSION = 0.6 - DEFAULT_SYSTEM_PROMPT = """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""" + # DEFAULT_SYSTEM_PROMPT = """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""" + DEFAULT_SYSTEM_PROMPT = "あなたは役立つアシスタントです。" SYSTEM_PROMPT = os.getenv("SYSTEM_PROMPT", DEFAULT_SYSTEM_PROMPT) DESCRIPTION = f"[INST] <>\n{SYSTEM_PROMPT}\n<>\n\n" INSTRUCTION = "与えられた5つの選択肢の中から、最適な答えを選んでください。" diff --git a/lm_eval/tasks/ja/jnli.py b/lm_eval/tasks/ja/jnli.py index 9339bbee13..069704c454 100644 --- a/lm_eval/tasks/ja/jnli.py +++ b/lm_eval/tasks/ja/jnli.py @@ -196,7 +196,8 @@ class JNLIWithLlama2(JNLIWithJAAlpacaPrompt): """ PROMPT_VERSION = 0.6 - DEFAULT_SYSTEM_PROMPT = """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""" + # DEFAULT_SYSTEM_PROMPT = """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""" + DEFAULT_SYSTEM_PROMPT = "あなたは役立つアシスタントです。" SYSTEM_PROMPT = os.getenv("SYSTEM_PROMPT", DEFAULT_SYSTEM_PROMPT) DESCRIPTION = f"[INST] <>\n{SYSTEM_PROMPT}\n<>\n\n" FEWSHOT_SEP = " [INST] " diff --git a/lm_eval/tasks/ja/jsquad.py b/lm_eval/tasks/ja/jsquad.py index 4ec544f654..347b99ebf7 100644 --- a/lm_eval/tasks/ja/jsquad.py +++ b/lm_eval/tasks/ja/jsquad.py @@ -381,7 +381,8 @@ class JSQuADWithLlama2(JSQuAD): """ PROMPT_VERSION = 0.6 - DEFAULT_SYSTEM_PROMPT = """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""" + # DEFAULT_SYSTEM_PROMPT = """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""" + DEFAULT_SYSTEM_PROMPT = "あなたは役立つアシスタントです。" SYSTEM_PROMPT = os.getenv("SYSTEM_PROMPT", DEFAULT_SYSTEM_PROMPT) DESCRIPTION = f"[INST] <>\n{SYSTEM_PROMPT}\n<>\n\n" INSTRUCTION = "与えられた文脈から、質問に対する答えを抜き出してください。" diff --git a/lm_eval/tasks/ja/marc_ja.py b/lm_eval/tasks/ja/marc_ja.py index 67ca22b048..a5ed0a25d1 100644 --- a/lm_eval/tasks/ja/marc_ja.py +++ b/lm_eval/tasks/ja/marc_ja.py @@ -171,7 +171,8 @@ class MARCJaWithLlama2(MARCJaWithJAAlpacaPrompt): """ PROMPT_VERSION = 0.6 - DEFAULT_SYSTEM_PROMPT = """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""" + # DEFAULT_SYSTEM_PROMPT = """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""" + DEFAULT_SYSTEM_PROMPT = "あなたは役立つアシスタントです。" SYSTEM_PROMPT = os.getenv("SYSTEM_PROMPT", DEFAULT_SYSTEM_PROMPT) DESCRIPTION = f"[INST] <>\n{SYSTEM_PROMPT}\n<>\n\n" FEWSHOT_SEP = " [INST] " diff --git a/lm_eval/tasks/ja/mgsm.py b/lm_eval/tasks/ja/mgsm.py index 18af141e43..2ccb12ae9a 100644 --- a/lm_eval/tasks/ja/mgsm.py +++ b/lm_eval/tasks/ja/mgsm.py @@ -178,7 +178,9 @@ class MGSMWithLlama2(MGSMWithJAAlpacaPrompt): """ PROMPT_VERSION = 0.6 - DEFAULT_SYSTEM_PROMPT = """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""" + # This is the default English prompt, and is included for reference. + # DEFAULT_SYSTEM_PROMPT = """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""" + DEFAULT_SYSTEM_PROMPT = "あなたは役立つアシスタントです。" SYSTEM_PROMPT = os.getenv("SYSTEM_PROMPT", DEFAULT_SYSTEM_PROMPT) DESCRIPTION = f"[INST] <>\n{SYSTEM_PROMPT}\n<>\n\n" FEWSHOT_SEP = " [INST] " diff --git a/lm_eval/tasks/ja/wikilingua.py b/lm_eval/tasks/ja/wikilingua.py index ab91cc2143..b0208bf0e5 100644 --- a/lm_eval/tasks/ja/wikilingua.py +++ b/lm_eval/tasks/ja/wikilingua.py @@ -179,7 +179,8 @@ class WikilinguaWithLlama2(Wikilingua): """ PROMPT_VERSION = 0.6 - DEFAULT_SYSTEM_PROMPT = """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""" + # DEFAULT_SYSTEM_PROMPT = """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""" + DEFAULT_SYSTEM_PROMPT = "あなたは役立つアシスタントです。" SYSTEM_PROMPT = os.getenv("SYSTEM_PROMPT", DEFAULT_SYSTEM_PROMPT) DESCRIPTION = f"[INST] <>\n{SYSTEM_PROMPT}\n<>\n\n" FEWSHOT_SEP = " [INST] " diff --git a/lm_eval/tasks/ja/xlsum_ja.py b/lm_eval/tasks/ja/xlsum_ja.py index 014193a89b..5c73acbcfb 100644 --- a/lm_eval/tasks/ja/xlsum_ja.py +++ b/lm_eval/tasks/ja/xlsum_ja.py @@ -252,7 +252,8 @@ class XLSumJaWithLlama2(XLSumJa): """ PROMPT_VERSION = 0.6 - DEFAULT_SYSTEM_PROMPT = """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""" + # DEFAULT_SYSTEM_PROMPT = """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""" + DEFAULT_SYSTEM_PROMPT = "あなたは役立つアシスタントです。" SYSTEM_PROMPT = os.getenv("SYSTEM_PROMPT", DEFAULT_SYSTEM_PROMPT) DESCRIPTION = f"[INST] <>\n{SYSTEM_PROMPT}\n<>\n\n" INSTRUCTION = "与えられたニュース記事を要約してください。"