dspy-phi3-copro.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<h2>Installing Ollama</h2>","metadata":{}},{"cell_type":"code","source":"!curl -fsSL https://ollama.com/install.sh | sh","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:43:01.595336Z","iopub.execute_input":"2024-06-16T08:43:01.595905Z","iopub.status.idle":"2024-06-16T08:43:04.851533Z","shell.execute_reply.started":"2024-06-16T08:43:01.595876Z","shell.execute_reply":"2024-06-16T08:43:04.850379Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":">>> Downloading ollama...\n######################################################################## 100.0%#=#=#                                                                          \n>>> Installing ollama to /usr/local/bin...\n>>> Creating ollama user...\n>>> Adding ollama user to video group...\n>>> Adding current user to ollama group...\n>>> Creating ollama systemd service...\n>>> NVIDIA GPU installed.\n>>> The Ollama API is now available at 127.0.0.1:11434.\n>>> Install complete. Run \"ollama\" from the command line.\n","output_type":"stream"}]},{"cell_type":"code","source":"# Thanks to:\n# https://www.kaggle.com/code/adriensales/ollama-running-local-models-w-llamaindex-cpu\nimport subprocess\nimport time\n# Start ollama as a backrgound process\ncommand = \"env OLLAMA_NUM_PARALLEL=3 nohup ollama serve&\"\n# Use subprocess.Popen to start the process in the background\nprocess = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\nprint(\"Process ID:\", process.pid)\ntime.sleep(5)  # Makes Python wait for 5 seconds","metadata":{"execution":{"iopub.status.busy":"2024-06-16T09:26:11.831980Z","iopub.execute_input":"2024-06-16T09:26:11.832741Z","iopub.status.idle":"2024-06-16T09:26:16.844864Z","shell.execute_reply.started":"2024-06-16T09:26:11.832683Z","shell.execute_reply":"2024-06-16T09:26:16.844046Z"},"trusted":true},"execution_count":48,"outputs":[{"name":"stdout","text":"Process ID: 948\n","output_type":"stream"}]},{"cell_type":"code","source":"!ollama pull phi3","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q datasets dspy-ai >/dev/null 2>&1","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:44:56.751437Z","iopub.execute_input":"2024-06-16T08:44:56.751876Z","iopub.status.idle":"2024-06-16T08:45:17.131747Z","shell.execute_reply.started":"2024-06-16T08:44:56.751836Z","shell.execute_reply":"2024-06-16T08:45:17.130486Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"markdown","source":"<h1>Get dataset</h1> Corpus of Linguistic Acceptability (CoLA) consists of sentences from 23 linguistics publications, expertly annotated for acceptability (grammaticality) ","metadata":{}},{"cell_type":"code","source":"from datasets import load_dataset\n\ndataset = load_dataset(\"linxinyuan/cola\")","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:45:17.133573Z","iopub.execute_input":"2024-06-16T08:45:17.133894Z","iopub.status.idle":"2024-06-16T08:45:21.310375Z","shell.execute_reply.started":"2024-06-16T08:45:17.133865Z","shell.execute_reply":"2024-06-16T08:45:21.309674Z"},"trusted":true},"execution_count":5,"outputs":[{"output_type":"display_data","data":{"text/plain":"Downloading builder script:   0%|          | 0.00/1.43k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"442ed74ddd36475da051199623d71414"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading data:   0%|          | 0.00/429k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"099f9c8250074658878c856bc3bebc90"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading data:   0%|          | 0.00/26.0k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3b4ecef34a184f82a4e8f19279c45640"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating train split: 0 examples [00:00, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"aa8849f04163454eb5408a17fe9b3e49"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating test split: 0 examples [00:00, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b4bd525524484909be06362a18c307cf"}},"metadata":{}}]},{"cell_type":"markdown","source":"<h2>Dspy - Prepare data</h2>","metadata":{}},{"cell_type":"code","source":"import dspy\nfrom dspy.teleprompt import COPRO\ntrain_ds=dataset['train'].shuffle()[:50]\nsentences,labels=train_ds['text'],train_ds['label']\ntrainset= [dspy.Example(sentence=sentence, answer=str(label)).with_inputs(\"sentence\") for sentence,label in zip(sentences,labels)]\ntest_ds=dataset['test'].shuffle()[:50]\nsentences_test,labels_test=test_ds['text'],test_ds['label']\ntestset= [dspy.Example(sentence=sentence, answer=str(label)).with_inputs(\"sentence\") for sentence,label in zip(sentences_test,labels_test)]\n","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:47:03.961207Z","iopub.execute_input":"2024-06-16T08:47:03.962108Z","iopub.status.idle":"2024-06-16T08:47:06.691201Z","shell.execute_reply.started":"2024-06-16T08:47:03.962072Z","shell.execute_reply":"2024-06-16T08:47:06.690237Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"code","source":"testset[:5]","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:47:24.955372Z","iopub.execute_input":"2024-06-16T08:47:24.956101Z","iopub.status.idle":"2024-06-16T08:47:24.962219Z","shell.execute_reply.started":"2024-06-16T08:47:24.956065Z","shell.execute_reply":"2024-06-16T08:47:24.961137Z"},"trusted":true},"execution_count":9,"outputs":[{"execution_count":9,"output_type":"execute_result","data":{"text/plain":"[Example({'sentence': 'This proved a decisive factor.', 'answer': '1'}) (input_keys={'sentence'}),\n Example({'sentence': 'he had spent five thousand dollars.', 'answer': '1'}) (input_keys={'sentence'}),\n Example({'sentence': 'Nora sent Peter the book.', 'answer': '1'}) (input_keys={'sentence'}),\n Example({'sentence': 'Henry found that Bill is sad.', 'answer': '1'}) (input_keys={'sentence'}),\n Example({'sentence': 'I found the place where we can relax.', 'answer': '1'}) (input_keys={'sentence'})]"},"metadata":{}}]},{"cell_type":"markdown","source":"<h2>Dspy - langauge model</h2>","metadata":{}},{"cell_type":"code","source":"lm = dspy.OllamaLocal(model='phi3',timeout_s=300)\ndspy.configure(lm=lm)","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:47:29.990451Z","iopub.execute_input":"2024-06-16T08:47:29.990830Z","iopub.status.idle":"2024-06-16T08:47:29.995433Z","shell.execute_reply.started":"2024-06-16T08:47:29.990798Z","shell.execute_reply":"2024-06-16T08:47:29.994387Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"markdown","source":"<h2>Dspy - Signature(unrefined) : our initial unrefined prompt. </h2>\n    <p>The idea is not to get the best results for classification, but to see if dspy will be able to give us a somewhat refined version.</p>","metadata":{}},{"cell_type":"code","source":"class GetLabel(dspy.Signature):\n    \"\"\"Check if sentence is correct(1) or not(0)\"\"\"\n\n    sentence = dspy.InputField(\n        prefix = \"Sentence:\",\n        desc = \"a sentence in English\",\n    )\n    answer = dspy.OutputField(\n        prefix =\"Predicted label:\",\n        desc=\"predicted label only (1 or 0 only)\"\n    )\n\nclass AnswerBot(dspy.Module):\n    def __init__(self, tools=None):\n        super().__init__()\n\n        self.generate_answer = dspy.Predict(GetLabel)\n\n    def forward(self, sentence):\n        prediction = self.generate_answer(sentence=f\"{sentence}\")\n        return dspy.Prediction(answer=prediction.answer)\n\nuncompiled_answer_bot = AnswerBot()\n","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:47:35.077363Z","iopub.execute_input":"2024-06-16T08:47:35.078373Z","iopub.status.idle":"2024-06-16T08:47:35.087839Z","shell.execute_reply.started":"2024-06-16T08:47:35.078335Z","shell.execute_reply":"2024-06-16T08:47:35.086835Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"code","source":"def validate_answer(example, pred, trace=None):\n\n    answer_match = example.answer.lower().strip() == pred.answer.lower().strip()\n\n    return answer_match","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:47:39.355388Z","iopub.execute_input":"2024-06-16T08:47:39.355714Z","iopub.status.idle":"2024-06-16T08:47:39.360315Z","shell.execute_reply.started":"2024-06-16T08:47:39.355689Z","shell.execute_reply":"2024-06-16T08:47:39.359452Z"},"trusted":true},"execution_count":12,"outputs":[]},{"cell_type":"code","source":"from dspy.evaluate import Evaluate\n\n# Set up the evaluator, which can be re-used in your code.\nevaluator = Evaluate(devset=testset, num_threads=1, display_progress=True, display_table=50)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:47:41.155580Z","iopub.execute_input":"2024-06-16T08:47:41.155945Z","iopub.status.idle":"2024-06-16T08:47:41.160169Z","shell.execute_reply.started":"2024-06-16T08:47:41.155915Z","shell.execute_reply":"2024-06-16T08:47:41.159268Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"markdown","source":"<h2>Evaluate uncompiled</h2>","metadata":{}},{"cell_type":"code","source":"# Launch evaluation.\nevaluator(uncompiled_answer_bot, metric=validate_answer)","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:48:12.350392Z","iopub.execute_input":"2024-06-16T08:48:12.351237Z","iopub.status.idle":"2024-06-16T08:49:13.405611Z","shell.execute_reply.started":"2024-06-16T08:48:12.351204Z","shell.execute_reply":"2024-06-16T08:49:13.404710Z"},"trusted":true},"execution_count":17,"outputs":[{"name":"stdout","text":"Average Metric: 4 / 50  (8.0): 100%|██████████| 50/50 [01:01<00:00,  1.22s/it] \n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"<pandas.io.formats.style.Styler at 0x7deabf185f90>","text/html":"<style type=\"text/css\">\n#T_aab17 th {\n  text-align: left;\n}\n#T_aab17 td {\n  text-align: left;\n}\n#T_aab17_row0_col0, #T_aab17_row0_col1, #T_aab17_row0_col2, #T_aab17_row0_col3, #T_aab17_row1_col0, #T_aab17_row1_col1, #T_aab17_row1_col2, #T_aab17_row1_col3, #T_aab17_row2_col0, #T_aab17_row2_col1, #T_aab17_row2_col2, #T_aab17_row2_col3, #T_aab17_row3_col0, #T_aab17_row3_col1, #T_aab17_row3_col2, #T_aab17_row3_col3, #T_aab17_row4_col0, #T_aab17_row4_col1, #T_aab17_row4_col2, #T_aab17_row4_col3, #T_aab17_row5_col0, #T_aab17_row5_col1, #T_aab17_row5_col2, #T_aab17_row5_col3, #T_aab17_row6_col0, #T_aab17_row6_col1, #T_aab17_row6_col2, #T_aab17_row6_col3, #T_aab17_row7_col0, #T_aab17_row7_col1, #T_aab17_row7_col2, #T_aab17_row7_col3, #T_aab17_row8_col0, #T_aab17_row8_col1, #T_aab17_row8_col2, #T_aab17_row8_col3, #T_aab17_row9_col0, #T_aab17_row9_col1, #T_aab17_row9_col2, #T_aab17_row9_col3, #T_aab17_row10_col0, #T_aab17_row10_col1, #T_aab17_row10_col2, #T_aab17_row10_col3, #T_aab17_row11_col0, #T_aab17_row11_col1, #T_aab17_row11_col2, #T_aab17_row11_col3, #T_aab17_row12_col0, #T_aab17_row12_col1, #T_aab17_row12_col2, #T_aab17_row12_col3, #T_aab17_row13_col0, #T_aab17_row13_col1, #T_aab17_row13_col2, #T_aab17_row13_col3, #T_aab17_row14_col0, #T_aab17_row14_col1, #T_aab17_row14_col2, #T_aab17_row14_col3, #T_aab17_row15_col0, #T_aab17_row15_col1, #T_aab17_row15_col2, #T_aab17_row15_col3, #T_aab17_row16_col0, #T_aab17_row16_col1, #T_aab17_row16_col2, #T_aab17_row16_col3, #T_aab17_row17_col0, #T_aab17_row17_col1, #T_aab17_row17_col2, #T_aab17_row17_col3, #T_aab17_row18_col0, #T_aab17_row18_col1, #T_aab17_row18_col2, #T_aab17_row18_col3, #T_aab17_row19_col0, #T_aab17_row19_col1, #T_aab17_row19_col2, #T_aab17_row19_col3, #T_aab17_row20_col0, #T_aab17_row20_col1, #T_aab17_row20_col2, #T_aab17_row20_col3, #T_aab17_row21_col0, #T_aab17_row21_col1, #T_aab17_row21_col2, #T_aab17_row21_col3, #T_aab17_row22_col0, #T_aab17_row22_col1, #T_aab17_row22_col2, #T_aab17_row22_col3, #T_aab17_row23_col0, #T_aab17_row23_col1, #T_aab17_row23_col2, #T_aab17_row23_col3, #T_aab17_row24_col0, #T_aab17_row24_col1, #T_aab17_row24_col2, #T_aab17_row24_col3, #T_aab17_row25_col0, #T_aab17_row25_col1, #T_aab17_row25_col2, #T_aab17_row25_col3, #T_aab17_row26_col0, #T_aab17_row26_col1, #T_aab17_row26_col2, #T_aab17_row26_col3, #T_aab17_row27_col0, #T_aab17_row27_col1, #T_aab17_row27_col2, #T_aab17_row27_col3, #T_aab17_row28_col0, #T_aab17_row28_col1, #T_aab17_row28_col2, #T_aab17_row28_col3, #T_aab17_row29_col0, #T_aab17_row29_col1, #T_aab17_row29_col2, #T_aab17_row29_col3, #T_aab17_row30_col0, #T_aab17_row30_col1, #T_aab17_row30_col2, #T_aab17_row30_col3, #T_aab17_row31_col0, #T_aab17_row31_col1, #T_aab17_row31_col2, #T_aab17_row31_col3, #T_aab17_row32_col0, #T_aab17_row32_col1, #T_aab17_row32_col2, #T_aab17_row32_col3, #T_aab17_row33_col0, #T_aab17_row33_col1, #T_aab17_row33_col2, #T_aab17_row33_col3, #T_aab17_row34_col0, #T_aab17_row34_col1, #T_aab17_row34_col2, #T_aab17_row34_col3, #T_aab17_row35_col0, #T_aab17_row35_col1, #T_aab17_row35_col2, #T_aab17_row35_col3, #T_aab17_row36_col0, #T_aab17_row36_col1, #T_aab17_row36_col2, #T_aab17_row36_col3, #T_aab17_row37_col0, #T_aab17_row37_col1, #T_aab17_row37_col2, #T_aab17_row37_col3, #T_aab17_row38_col0, #T_aab17_row38_col1, #T_aab17_row38_col2, #T_aab17_row38_col3, #T_aab17_row39_col0, #T_aab17_row39_col1, #T_aab17_row39_col2, #T_aab17_row39_col3, #T_aab17_row40_col0, #T_aab17_row40_col1, #T_aab17_row40_col2, #T_aab17_row40_col3, #T_aab17_row41_col0, #T_aab17_row41_col1, #T_aab17_row41_col2, #T_aab17_row41_col3, #T_aab17_row42_col0, #T_aab17_row42_col1, #T_aab17_row42_col2, #T_aab17_row42_col3, #T_aab17_row43_col0, #T_aab17_row43_col1, #T_aab17_row43_col2, #T_aab17_row43_col3, #T_aab17_row44_col0, #T_aab17_row44_col1, #T_aab17_row44_col2, #T_aab17_row44_col3, #T_aab17_row45_col0, #T_aab17_row45_col1, #T_aab17_row45_col2, #T_aab17_row45_col3, #T_aab17_row46_col0, #T_aab17_row46_col1, #T_aab17_row46_col2, #T_aab17_row46_col3, #T_aab17_row47_col0, #T_aab17_row47_col1, #T_aab17_row47_col2, #T_aab17_row47_col3, #T_aab17_row48_col0, #T_aab17_row48_col1, #T_aab17_row48_col2, #T_aab17_row48_col3, #T_aab17_row49_col0, #T_aab17_row49_col1, #T_aab17_row49_col2, #T_aab17_row49_col3 {\n  text-align: left;\n  white-space: pre-wrap;\n  word-wrap: break-word;\n  max-width: 400px;\n}\n</style>\n<table id=\"T_aab17\">\n  <thead>\n    <tr>\n      <th class=\"blank level0\" >&nbsp;</th>\n      <th id=\"T_aab17_level0_col0\" class=\"col_heading level0 col0\" >sentence</th>\n      <th id=\"T_aab17_level0_col1\" class=\"col_heading level0 col1\" >example_answer</th>\n      <th id=\"T_aab17_level0_col2\" class=\"col_heading level0 col2\" >pred_answer</th>\n      <th id=\"T_aab17_level0_col3\" class=\"col_heading level0 col3\" >validate_answer</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th id=\"T_aab17_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n      <td id=\"T_aab17_row0_col0\" class=\"data row0 col0\" >This proved a decisive factor.</td>\n      <td id=\"T_aab17_row0_col1\" class=\"data row0 col1\" >1</td>\n      <td id=\"T_aab17_row0_col2\" class=\"data row0 col2\" >Predicted label: 1\n\nThe sentence \"This proved a decisive factor.\" is grammatically correct and conveys a clear meaning, so the predicted label is 1 (correct).</td>\n      <td id=\"T_aab17_row0_col3\" class=\"data row0 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n      <td id=\"T_aab17_row1_col0\" class=\"data row1 col0\" >he had spent five thousand dollars.</td>\n      <td id=\"T_aab17_row1_col1\" class=\"data row1 col1\" >1</td>\n      <td id=\"T_aab17_row1_col2\" class=\"data row1 col2\" >Predicted label: 1\n\nThe sentence \"he had spent five thousand dollars.\" is grammatically correct and makes sense, so the predicted label is 1.</td>\n      <td id=\"T_aab17_row1_col3\" class=\"data row1 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n      <td id=\"T_aab17_row2_col0\" class=\"data row2 col0\" >Nora sent Peter the book.</td>\n      <td id=\"T_aab17_row2_col1\" class=\"data row2 col1\" >1</td>\n      <td id=\"T_aab17_row2_col2\" class=\"data row2 col2\" >Predicted label: 1 The sentence \"Nora sent Peter the book.\" is grammatically correct, with a clear subject (Nora), verb (sent), and object (the book). The...</td>\n      <td id=\"T_aab17_row2_col3\" class=\"data row2 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n      <td id=\"T_aab17_row3_col0\" class=\"data row3 col0\" >Henry found that Bill is sad.</td>\n      <td id=\"T_aab17_row3_col1\" class=\"data row3 col1\" >1</td>\n      <td id=\"T_aab17_row3_col2\" class=\"data row3 col2\" >Predicted label: 1 The sentence \"Henry found that Bill is sad.\" is grammatically correct, with a clear subject (\"Henry\"), verb (\"found\"), and complementary clause expressing...</td>\n      <td id=\"T_aab17_row3_col3\" class=\"data row3 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n      <td id=\"T_aab17_row4_col0\" class=\"data row4 col0\" >I found the place where we can relax.</td>\n      <td id=\"T_aab17_row4_col1\" class=\"data row4 col1\" >1</td>\n      <td id=\"T_aab17_row4_col2\" class=\"data row4 col2\" >Predicted label: 1 The sentence \"I found the place where we can relax.\" is grammatically correct and makes sense, so it receives a predicted label...</td>\n      <td id=\"T_aab17_row4_col3\" class=\"data row4 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row5\" class=\"row_heading level0 row5\" >5</th>\n      <td id=\"T_aab17_row5_col0\" class=\"data row5 col0\" >I know which book Mag read, and which book Bob said that you hadn't.</td>\n      <td id=\"T_aab17_row5_col1\" class=\"data row5 col1\" >1</td>\n      <td id=\"T_aab17_row5_col2\" class=\"data row5 col2\" >Predicted label: 0 The sentence \"I know which book Mag read, and which book Bob said that you hadn't.\" contains a grammatical error with the...</td>\n      <td id=\"T_aab17_row5_col3\" class=\"data row5 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row6\" class=\"row_heading level0 row6\" >6</th>\n      <td id=\"T_aab17_row6_col0\" class=\"data row6 col0\" >Kim alienated cats and beating his dog.</td>\n      <td id=\"T_aab17_row6_col1\" class=\"data row6 col1\" >0</td>\n      <td id=\"T_aab17_row6_col2\" class=\"data row6 col2\" >Predicted label: 0 The sentence \"Kim alienated cats and beating his dog.\" is not correct due to grammatical errors, specifically the lack of a verb...</td>\n      <td id=\"T_aab17_row6_col3\" class=\"data row6 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row7\" class=\"row_heading level0 row7\" >7</th>\n      <td id=\"T_aab17_row7_col0\" class=\"data row7 col0\" >Most people probably consider, even though the courts didn't actually find, Klaus guilty of murder.</td>\n      <td id=\"T_aab17_row7_col1\" class=\"data row7 col1\" >1</td>\n      <td id=\"T_aab17_row7_col2\" class=\"data row7 col2\" >1 The given sentence is grammatically correct and conveys a clear meaning. It discusses an assumption about public opinion regarding a court decision involving a...</td>\n      <td id=\"T_aab17_row7_col3\" class=\"data row7 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row8\" class=\"row_heading level0 row8\" >8</th>\n      <td id=\"T_aab17_row8_col0\" class=\"data row8 col0\" >Came right in he did without so much as a knock.</td>\n      <td id=\"T_aab17_row8_col1\" class=\"data row8 col1\" >1</td>\n      <td id=\"T_aab17_row8_col2\" class=\"data row8 col2\" >1 The sentence \"Came right in he did without so much as a knock.\" is grammatically correct, although it may sound slightly informal or conversational....</td>\n      <td id=\"T_aab17_row8_col3\" class=\"data row8 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row9\" class=\"row_heading level0 row9\" >9</th>\n      <td id=\"T_aab17_row9_col0\" class=\"data row9 col0\" >We pulled free.</td>\n      <td id=\"T_aab17_row9_col1\" class=\"data row9 col1\" >1</td>\n      <td id=\"T_aab17_row9_col2\" class=\"data row9 col2\" >Sentence: We pulled free.\nPredicted label: 1</td>\n      <td id=\"T_aab17_row9_col3\" class=\"data row9 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row10\" class=\"row_heading level0 row10\" >10</th>\n      <td id=\"T_aab17_row10_col0\" class=\"data row10 col0\" >Sarah promised Catherine her old car, but then gave it to her son instead.</td>\n      <td id=\"T_aab17_row10_col1\" class=\"data row10 col1\" >1</td>\n      <td id=\"T_aab17_row10_col2\" class=\"data row10 col2\" >1</td>\n      <td id=\"T_aab17_row10_col3\" class=\"data row10 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row11\" class=\"row_heading level0 row11\" >11</th>\n      <td id=\"T_aab17_row11_col0\" class=\"data row11 col0\" >After reading the pamphlet, Judy threw it into the garbage can.</td>\n      <td id=\"T_aab17_row11_col1\" class=\"data row11 col1\" >1</td>\n      <td id=\"T_aab17_row11_col2\" class=\"data row11 col2\" >1</td>\n      <td id=\"T_aab17_row11_col3\" class=\"data row11 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row12\" class=\"row_heading level0 row12\" >12</th>\n      <td id=\"T_aab17_row12_col0\" class=\"data row12 col0\" >Kim put in the box.</td>\n      <td id=\"T_aab17_row12_col1\" class=\"data row12 col1\" >0</td>\n      <td id=\"T_aab17_row12_col2\" class=\"data row12 col2\" >Predicted label: 1\n\n(Note: The sentence \"Kim put in the box.\" is grammatically correct, so the predicted label is 1.)</td>\n      <td id=\"T_aab17_row12_col3\" class=\"data row12 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row13\" class=\"row_heading level0 row13\" >13</th>\n      <td id=\"T_aab17_row13_col0\" class=\"data row13 col0\" >John tried to please Stephen.</td>\n      <td id=\"T_aab17_row13_col1\" class=\"data row13 col1\" >1</td>\n      <td id=\"T_aab17_row13_col2\" class=\"data row13 col2\" >Predicted label: 1\n\nThe sentence \"John tried to please Stephen.\" is grammatically correct and makes sense, so the predicted label is 1.</td>\n      <td id=\"T_aab17_row13_col3\" class=\"data row13 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row14\" class=\"row_heading level0 row14\" >14</th>\n      <td id=\"T_aab17_row14_col0\" class=\"data row14 col0\" >It was the peasant girl who got it.</td>\n      <td id=\"T_aab17_row14_col1\" class=\"data row14 col1\" >1</td>\n      <td id=\"T_aab17_row14_col2\" class=\"data row14 col2\" >Predicted label: 1 The sentence \"It was the peasant girl who got it.\" is grammatically correct and makes sense, so the predicted label is 1...</td>\n      <td id=\"T_aab17_row14_col3\" class=\"data row14 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row15\" class=\"row_heading level0 row15\" >15</th>\n      <td id=\"T_aab17_row15_col0\" class=\"data row15 col0\" >Anson believed himself to be handsome.</td>\n      <td id=\"T_aab17_row15_col1\" class=\"data row15 col1\" >1</td>\n      <td id=\"T_aab17_row15_col2\" class=\"data row15 col2\" >Predicted label: 1\n\nThe sentence \"Anson believed himself to be handsome.\" is grammatically correct and makes sense, so the predicted label is 1 (correct).</td>\n      <td id=\"T_aab17_row15_col3\" class=\"data row15 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row16\" class=\"row_heading level0 row16\" >16</th>\n      <td id=\"T_aab17_row16_col0\" class=\"data row16 col0\" >This girl in the red coat will put a picture of Bill in the mailbox and on your desk before tomorrow.</td>\n      <td id=\"T_aab17_row16_col1\" class=\"data row16 col1\" >1</td>\n      <td id=\"T_aab17_row16_col2\" class=\"data row16 col2\" >Predicted label: 0 The sentence seems incorrect because it suggests that someone (presumably not mentioned earlier) is putting a picture of \"Bill\" somewhere, which might...</td>\n      <td id=\"T_aab17_row16_col3\" class=\"data row16 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row17\" class=\"row_heading level0 row17\" >17</th>\n      <td id=\"T_aab17_row17_col0\" class=\"data row17 col0\" >Mike talked about politics yesterday to my friends.</td>\n      <td id=\"T_aab17_row17_col1\" class=\"data row17 col1\" >1</td>\n      <td id=\"T_aab17_row17_col2\" class=\"data row17 col2\" >Predicted label: 1 The sentence \"Mike talked about politics yesterday to my friends.\" is grammatically correct and makes sense, so the predicted label is 1...</td>\n      <td id=\"T_aab17_row17_col3\" class=\"data row17 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row18\" class=\"row_heading level0 row18\" >18</th>\n      <td id=\"T_aab17_row18_col0\" class=\"data row18 col0\" >John is eager.</td>\n      <td id=\"T_aab17_row18_col1\" class=\"data row18 col1\" >1</td>\n      <td id=\"T_aab17_row18_col2\" class=\"data row18 col2\" >Predicted label: 1\n\nThe sentence \"John is eager.\" is grammatically correct and conveys a clear meaning, so the predicted label is 1.</td>\n      <td id=\"T_aab17_row18_col3\" class=\"data row18 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row19\" class=\"row_heading level0 row19\" >19</th>\n      <td id=\"T_aab17_row19_col0\" class=\"data row19 col0\" >Any owl hunts mice.</td>\n      <td id=\"T_aab17_row19_col1\" class=\"data row19 col1\" >1</td>\n      <td id=\"T_aab17_row19_col2\" class=\"data row19 col2\" >Predicted label: 1\n\nThe sentence \"Any owl hunts mice.\" is grammatically correct and conveys a clear meaning, so the predicted label is 1 (correct).</td>\n      <td id=\"T_aab17_row19_col3\" class=\"data row19 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row20\" class=\"row_heading level0 row20\" >20</th>\n      <td id=\"T_aab17_row20_col0\" class=\"data row20 col0\" >The car was driven.</td>\n      <td id=\"T_aab17_row20_col1\" class=\"data row20 col1\" >1</td>\n      <td id=\"T_aab17_row20_col2\" class=\"data row20 col2\" >Predicted label: 1 The sentence \"The car was driven.\" is grammatically correct, as it follows the standard subject-verb-object structure in English and uses the passive...</td>\n      <td id=\"T_aab17_row20_col3\" class=\"data row20 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row21\" class=\"row_heading level0 row21\" >21</th>\n      <td id=\"T_aab17_row21_col0\" class=\"data row21 col0\" >We linguists love to argue</td>\n      <td id=\"T_aab17_row21_col1\" class=\"data row21 col1\" >1</td>\n      <td id=\"T_aab17_row21_col2\" class=\"data row21 col2\" >Predicted label: 1\n\nThe sentence \"We linguists love to argue\" is grammatically correct and makes sense, so the predicted label is 1.</td>\n      <td id=\"T_aab17_row21_col3\" class=\"data row21 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row22\" class=\"row_heading level0 row22\" >22</th>\n      <td id=\"T_aab17_row22_col0\" class=\"data row22 col0\" >John often meets Mary.</td>\n      <td id=\"T_aab17_row22_col1\" class=\"data row22 col1\" >1</td>\n      <td id=\"T_aab17_row22_col2\" class=\"data row22 col2\" >Predicted label: 1\n\nThe sentence \"John often meets Mary.\" is grammatically correct and conveys a clear meaning, so the predicted label is 1.</td>\n      <td id=\"T_aab17_row22_col3\" class=\"data row22 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row23\" class=\"row_heading level0 row23\" >23</th>\n      <td id=\"T_aab17_row23_col0\" class=\"data row23 col0\" >The farmer loaded the cart with apples.</td>\n      <td id=\"T_aab17_row23_col1\" class=\"data row23 col1\" >1</td>\n      <td id=\"T_aab17_row23_col2\" class=\"data row23 col2\" >1</td>\n      <td id=\"T_aab17_row23_col3\" class=\"data row23 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row24\" class=\"row_heading level0 row24\" >24</th>\n      <td id=\"T_aab17_row24_col0\" class=\"data row24 col0\" >The government's imposition of a fine.</td>\n      <td id=\"T_aab17_row24_col1\" class=\"data row24 col1\" >1</td>\n      <td id=\"T_aab17_row24_col2\" class=\"data row24 col2\" >Sentence: The government's imposition of a fine. Predicted label: 0 Explanation: The sentence \"The government's imposition of a fine.\" is not complete as it lacks...</td>\n      <td id=\"T_aab17_row24_col3\" class=\"data row24 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row25\" class=\"row_heading level0 row25\" >25</th>\n      <td id=\"T_aab17_row25_col0\" class=\"data row25 col0\" >Mickey looked up it.</td>\n      <td id=\"T_aab17_row25_col1\" class=\"data row25 col1\" >0</td>\n      <td id=\"T_aab17_row25_col2\" class=\"data row25 col2\" >Predicted label: 0 The correct form of the sentence should be \"Mickey looked up at it.\" The original sentence is incorrect due to missing preposition...</td>\n      <td id=\"T_aab17_row25_col3\" class=\"data row25 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row26\" class=\"row_heading level0 row26\" >26</th>\n      <td id=\"T_aab17_row26_col0\" class=\"data row26 col0\" >No writer, and no playwright, meets in Vienna.</td>\n      <td id=\"T_aab17_row26_col1\" class=\"data row26 col1\" >0</td>\n      <td id=\"T_aab17_row26_col2\" class=\"data row26 col2\" >Predicted label: 0 Explanation: The sentence \"No writer, and no playwright, meets in Vienna.\" is incorrect because it suggests that none of the writers or...</td>\n      <td id=\"T_aab17_row26_col3\" class=\"data row26 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row27\" class=\"row_heading level0 row27\" >27</th>\n      <td id=\"T_aab17_row27_col0\" class=\"data row27 col0\" >Frank Churchill crossed the street.</td>\n      <td id=\"T_aab17_row27_col1\" class=\"data row27 col1\" >1</td>\n      <td id=\"T_aab17_row27_col2\" class=\"data row27 col2\" >Predicted label: 1\n\nThe sentence \"Frank Churchill crossed the street.\" is grammatically correct and makes sense, so it should be labeled as 1 for correct.</td>\n      <td id=\"T_aab17_row27_col3\" class=\"data row27 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row28\" class=\"row_heading level0 row28\" >28</th>\n      <td id=\"T_aab17_row28_col0\" class=\"data row28 col0\" >Suddenly, there arrived two inspectors from the INS.</td>\n      <td id=\"T_aab17_row28_col1\" class=\"data row28 col1\" >1</td>\n      <td id=\"T_aab17_row28_col2\" class=\"data row28 col2\" >1 The sentence \"Suddenly, there arrived two inspectors from the INS.\" is grammatically correct and conveys a clear meaning, so it receives a predicted label...</td>\n      <td id=\"T_aab17_row28_col3\" class=\"data row28 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row29\" class=\"row_heading level0 row29\" >29</th>\n      <td id=\"T_aab17_row29_col0\" class=\"data row29 col0\" >I'm sure we even got these tickets!</td>\n      <td id=\"T_aab17_row29_col1\" class=\"data row29 col1\" >1</td>\n      <td id=\"T_aab17_row29_col2\" class=\"data row29 col2\" >Predicted label: 1\n\nThe sentence \"I'm sure we even got these tickets!\" is grammatically correct and makes sense, so the predicted label is 1.</td>\n      <td id=\"T_aab17_row29_col3\" class=\"data row29 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row30\" class=\"row_heading level0 row30\" >30</th>\n      <td id=\"T_aab17_row30_col0\" class=\"data row30 col0\" >One of Korea's most famous poets wrote these lines.</td>\n      <td id=\"T_aab17_row30_col1\" class=\"data row30 col1\" >1</td>\n      <td id=\"T_aab17_row30_col2\" class=\"data row30 col2\" >1</td>\n      <td id=\"T_aab17_row30_col3\" class=\"data row30 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row31\" class=\"row_heading level0 row31\" >31</th>\n      <td id=\"T_aab17_row31_col0\" class=\"data row31 col0\" >Heidi thinks that Andy should eat salmon flavored candy bars.</td>\n      <td id=\"T_aab17_row31_col1\" class=\"data row31 col1\" >1</td>\n      <td id=\"T_aab17_row31_col2\" class=\"data row31 col2\" >Predicted label: 0 The sentence \"Heidi thinks that Andy should eat salmon flavored candy bars\" is not correct in a typical context because it's unusual...</td>\n      <td id=\"T_aab17_row31_col3\" class=\"data row31 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row32\" class=\"row_heading level0 row32\" >32</th>\n      <td id=\"T_aab17_row32_col0\" class=\"data row32 col0\" >The box contained the ball from the tree.</td>\n      <td id=\"T_aab17_row32_col1\" class=\"data row32 col1\" >0</td>\n      <td id=\"T_aab17_row32_col2\" class=\"data row32 col2\" >0 (Note: The sentence \"The box contained the ball from the tree.\" is likely incorrect due to its implausibility, as it's unusual for a ball...</td>\n      <td id=\"T_aab17_row32_col3\" class=\"data row32 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row33\" class=\"row_heading level0 row33\" >33</th>\n      <td id=\"T_aab17_row33_col0\" class=\"data row33 col0\" >A job offered.</td>\n      <td id=\"T_aab17_row33_col1\" class=\"data row33 col1\" >0</td>\n      <td id=\"T_aab17_row33_col2\" class=\"data row33 col2\" >Predicted label: 0 The given sentence \"A job offered.\" is not complete as it lacks context and details to be considered correct in standard English...</td>\n      <td id=\"T_aab17_row33_col3\" class=\"data row33 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row34\" class=\"row_heading level0 row34\" >34</th>\n      <td id=\"T_aab17_row34_col0\" class=\"data row34 col0\" >I want to try and buy some whiskey.</td>\n      <td id=\"T_aab17_row34_col1\" class=\"data row34 col1\" >1</td>\n      <td id=\"T_aab17_row34_col2\" class=\"data row34 col2\" >Sentence: I want to try and buy some whiskey. Predicted label: 1 The sentence is grammatically correct, although it may imply the speaker intends to...</td>\n      <td id=\"T_aab17_row34_col3\" class=\"data row34 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row35\" class=\"row_heading level0 row35\" >35</th>\n      <td id=\"T_aab17_row35_col0\" class=\"data row35 col0\" >Jackie accompanied Rose.</td>\n      <td id=\"T_aab17_row35_col1\" class=\"data row35 col1\" >1</td>\n      <td id=\"T_aab17_row35_col2\" class=\"data row35 col2\" >Predicted label: 1 The sentence \"Jackie accompanied Rose.\" is grammatically correct, as it properly conveys that Jackie was with Rose in some capacity. Therefore, the...</td>\n      <td id=\"T_aab17_row35_col3\" class=\"data row35 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row36\" class=\"row_heading level0 row36\" >36</th>\n      <td id=\"T_aab17_row36_col0\" class=\"data row36 col0\" >Every student, and he wears socks, is a swinger.</td>\n      <td id=\"T_aab17_row36_col1\" class=\"data row36 col1\" >0</td>\n      <td id=\"T_aab17_row36_col2\" class=\"data row36 col2\" >Predicted label: 0 The sentence \"Every student, and he wears socks, is a swinger.\" seems incorrect due to the improbable connection between students wearing socks...</td>\n      <td id=\"T_aab17_row36_col3\" class=\"data row36 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row37\" class=\"row_heading level0 row37\" >37</th>\n      <td id=\"T_aab17_row37_col0\" class=\"data row37 col0\" >That the fuzz wanted him worried John, but that the fuzz wanted her didn't worry Mary.</td>\n      <td id=\"T_aab17_row37_col1\" class=\"data row37 col1\" >1</td>\n      <td id=\"T_aab17_row37_col2\" class=\"data row37 col2\" >Predicted label: 0 The sentence has grammatical issues and is not correctly structured in English. A corrected version could be: \"That the fuzz was worried...</td>\n      <td id=\"T_aab17_row37_col3\" class=\"data row37 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row38\" class=\"row_heading level0 row38\" >38</th>\n      <td id=\"T_aab17_row38_col0\" class=\"data row38 col0\" >Drowning cats, which is against the law, are hard to rescue.</td>\n      <td id=\"T_aab17_row38_col1\" class=\"data row38 col1\" >0</td>\n      <td id=\"T_aab17_row38_col2\" class=\"data row38 col2\" >Predicted label: 0 The sentence \"Drowning cats, which is against the law, are hard to rescue.\" contains a subject-verb agreement error. The correct form should...</td>\n      <td id=\"T_aab17_row38_col3\" class=\"data row38 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row39\" class=\"row_heading level0 row39\" >39</th>\n      <td id=\"T_aab17_row39_col0\" class=\"data row39 col0\" >We believed John to be a fountain in the park.</td>\n      <td id=\"T_aab17_row39_col1\" class=\"data row39 col1\" >0</td>\n      <td id=\"T_aab17_row39_col2\" class=\"data row39 col2\" >Predicted label: 0 The sentence \"We believed John to be a fountain in the park.\" is not correct because it uses an unusual metaphor that...</td>\n      <td id=\"T_aab17_row39_col3\" class=\"data row39 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row40\" class=\"row_heading level0 row40\" >40</th>\n      <td id=\"T_aab17_row40_col0\" class=\"data row40 col0\" >This flyer and that flyer differ.</td>\n      <td id=\"T_aab17_row40_col1\" class=\"data row40 col1\" >1</td>\n      <td id=\"T_aab17_row40_col2\" class=\"data row40 col2\" >Predicted label: 1 The sentence \"This flyer and that flyer differ.\" is grammatically correct, as it properly conveys the meaning that there are differences between...</td>\n      <td id=\"T_aab17_row40_col3\" class=\"data row40 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row41\" class=\"row_heading level0 row41\" >41</th>\n      <td id=\"T_aab17_row41_col0\" class=\"data row41 col0\" >The man I saw get into the cab robbed the bank.</td>\n      <td id=\"T_aab17_row41_col1\" class=\"data row41 col1\" >1</td>\n      <td id=\"T_aab17_row41_col2\" class=\"data row41 col2\" >Predicted label: 0 The sentence \"The man I saw get into the cab robbed the bank.\" is not correct because it implies that the act...</td>\n      <td id=\"T_aab17_row41_col3\" class=\"data row41 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row42\" class=\"row_heading level0 row42\" >42</th>\n      <td id=\"T_aab17_row42_col0\" class=\"data row42 col0\" >This teacher is a genius.</td>\n      <td id=\"T_aab17_row42_col1\" class=\"data row42 col1\" >1</td>\n      <td id=\"T_aab17_row42_col2\" class=\"data row42 col2\" >Predicted label: 1\n\nThe sentence \"This teacher is a genius.\" is grammatically correct and conveys a clear meaning, so the predicted label is 1 (correct).</td>\n      <td id=\"T_aab17_row42_col3\" class=\"data row42 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row43\" class=\"row_heading level0 row43\" >43</th>\n      <td id=\"T_aab17_row43_col0\" class=\"data row43 col0\" >Linda taped the picture to the wall.</td>\n      <td id=\"T_aab17_row43_col1\" class=\"data row43 col1\" >1</td>\n      <td id=\"T_aab17_row43_col2\" class=\"data row43 col2\" >Predicted label: 1 The sentence \"Linda taped the picture to the wall.\" is grammatically correct and makes sense, so it should be labeled as (1)...</td>\n      <td id=\"T_aab17_row43_col3\" class=\"data row43 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row44\" class=\"row_heading level0 row44\" >44</th>\n      <td id=\"T_aab17_row44_col0\" class=\"data row44 col0\" >I live at the place where Route 150 crosses the Hudson River and my dad lives at it too.</td>\n      <td id=\"T_aab17_row44_col1\" class=\"data row44 col1\" >0</td>\n      <td id=\"T_aab17_row44_col2\" class=\"data row44 col2\" >Predicted label: 1 Explanation: The sentence is grammatically correct, although \"it\" might be considered ambiguous without further context. However, based on the information provided, there's...</td>\n      <td id=\"T_aab17_row44_col3\" class=\"data row44 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row45\" class=\"row_heading level0 row45\" >45</th>\n      <td id=\"T_aab17_row45_col0\" class=\"data row45 col0\" >Have you seen my model airplane collection?</td>\n      <td id=\"T_aab17_row45_col1\" class=\"data row45 col1\" >1</td>\n      <td id=\"T_aab17_row45_col2\" class=\"data row45 col2\" >Sentence: Have you seen my model airplane collection?\nPredicted label: 1</td>\n      <td id=\"T_aab17_row45_col3\" class=\"data row45 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row46\" class=\"row_heading level0 row46\" >46</th>\n      <td id=\"T_aab17_row46_col0\" class=\"data row46 col0\" >What did you ask who Medea gave?</td>\n      <td id=\"T_aab17_row46_col1\" class=\"data row46 col1\" >0</td>\n      <td id=\"T_aab17_row46_col2\" class=\"data row46 col2\" >Sentence: What did you ask who Medea gave? Predicted label: 0 The sentence \"What did you ask who Medea gave?\" is not correct in standard...</td>\n      <td id=\"T_aab17_row46_col3\" class=\"data row46 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row47\" class=\"row_heading level0 row47\" >47</th>\n      <td id=\"T_aab17_row47_col0\" class=\"data row47 col0\" >Who did John send the book?</td>\n      <td id=\"T_aab17_row47_col1\" class=\"data row47 col1\" >1</td>\n      <td id=\"T_aab17_row47_col2\" class=\"data row47 col2\" >Sentence: Who did John send the book? Predicted label: 0 The given sentence is grammatically incorrect. A more correct version would be \"Who did John...</td>\n      <td id=\"T_aab17_row47_col3\" class=\"data row47 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row48\" class=\"row_heading level0 row48\" >48</th>\n      <td id=\"T_aab17_row48_col0\" class=\"data row48 col0\" >Carla shoveled the walk.</td>\n      <td id=\"T_aab17_row48_col1\" class=\"data row48 col1\" >1</td>\n      <td id=\"T_aab17_row48_col2\" class=\"data row48 col2\" >Predicted label: 1 The sentence \"Carla shoveled the walk.\" is grammatically correct, although it might sound unusual because typically one would say \"shovel snow\" or...</td>\n      <td id=\"T_aab17_row48_col3\" class=\"data row48 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_aab17_level0_row49\" class=\"row_heading level0 row49\" >49</th>\n      <td id=\"T_aab17_row49_col0\" class=\"data row49 col0\" >I know you eat asparagus.</td>\n      <td id=\"T_aab17_row49_col1\" class=\"data row49 col1\" >1</td>\n      <td id=\"T_aab17_row49_col2\" class=\"data row49 col2\" >Sentence: I know you eat asparagus. Predicted label: 1 The sentence \"I know you eat asparagus.\" is grammatically correct, assuming the context in which it's...</td>\n      <td id=\"T_aab17_row49_col3\" class=\"data row49 col3\" >False</td>\n    </tr>\n  </tbody>\n</table>\n"},"metadata":{}},{"execution_count":17,"output_type":"execute_result","data":{"text/plain":"8.0"},"metadata":{}}]},{"cell_type":"markdown","source":"<h4>The responses are not bad but the score is. We just need label without any explanation and without any extra words/labels</h4>","metadata":{}},{"cell_type":"code","source":"lm.history[-1]['prompt']","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:49:23.030363Z","iopub.execute_input":"2024-06-16T08:49:23.031331Z","iopub.status.idle":"2024-06-16T08:49:23.037852Z","shell.execute_reply.started":"2024-06-16T08:49:23.031295Z","shell.execute_reply":"2024-06-16T08:49:23.036977Z"},"trusted":true},"execution_count":18,"outputs":[{"execution_count":18,"output_type":"execute_result","data":{"text/plain":"'Check if sentence is correct(1) or not(0)\\n\\n---\\n\\nFollow the following format.\\n\\nSentence: a sentence in English\\nPredicted label: predicted label only (1 or 0 only)\\n\\n---\\n\\nSentence: I know you eat asparagus.\\nPredicted label:'"},"metadata":{}}]},{"cell_type":"markdown","source":"<h2>Let's see if we can get a better instruction</h2>","metadata":{}},{"cell_type":"code","source":"teleprompter = COPRO(metric=validate_answer,depth=2)\nkwargs = dict(num_threads=64, display_progress=True) \ncompiled_answer_bot = teleprompter.compile(uncompiled_answer_bot, trainset=trainset,eval_kwargs=kwargs)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-06-15T10:22:20.266778Z","iopub.execute_input":"2024-06-15T10:22:20.267173Z","iopub.status.idle":"2024-06-15T10:28:14.023783Z","shell.execute_reply.started":"2024-06-15T10:22:20.267143Z","shell.execute_reply":"2024-06-15T10:28:14.022953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"compiled_answer_bot.candidate_programs","metadata":{"execution":{"iopub.status.busy":"2024-06-15T10:28:21.610533Z","iopub.execute_input":"2024-06-15T10:28:21.610896Z","iopub.status.idle":"2024-06-15T10:28:21.620278Z","shell.execute_reply.started":"2024-06-15T10:28:21.610868Z","shell.execute_reply":"2024-06-15T10:28:21.619447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h2>Define better signature</h2> by trying the best instructions from the above proposed instructions</h2>","metadata":{}},{"cell_type":"code","source":"class GetLabelOptimized(dspy.Signature):\n    \"\"\"Determine if the given sentence adheres to standard English grammar rules by outputting a '1' for grammatically correct and '0' for incorrect.\"\"\"\n\n    sentence = dspy.InputField(\n        prefix = \"Sentence:\",\n        desc = \"a sentence in English\",\n    )\n    answer = dspy.OutputField(\n        prefix=\"\"\"If the input sentence is grammatically correct, please respond with 1; otherwise, respond with 0.\"\"\",\n        desc=\"predicted label only (1 or 0 only)\"\n    )\n\nclass AnswerBotOptimized(dspy.Module):\n    def __init__(self, tools=None):\n        super().__init__()\n\n        self.generate_answer = dspy.Predict(GetLabelOptimized)\n\n    def forward(self, sentence):\n        prediction = self.generate_answer(sentence=f\"{sentence}\")\n        return dspy.Prediction(answer=prediction.answer)","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:47:46.075872Z","iopub.execute_input":"2024-06-16T08:47:46.076213Z","iopub.status.idle":"2024-06-16T08:47:46.084924Z","shell.execute_reply.started":"2024-06-16T08:47:46.076189Z","shell.execute_reply":"2024-06-16T08:47:46.083954Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"optimized_answer_bot = AnswerBotOptimized()","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:47:49.150611Z","iopub.execute_input":"2024-06-16T08:47:49.151258Z","iopub.status.idle":"2024-06-16T08:47:49.155186Z","shell.execute_reply.started":"2024-06-16T08:47:49.151224Z","shell.execute_reply":"2024-06-16T08:47:49.154360Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"markdown","source":"<h2>Let's test</h2>","metadata":{}},{"cell_type":"code","source":"# Launch evaluation.\nevaluator(optimized_answer_bot, metric=validate_answer)","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:49:27.510823Z","iopub.execute_input":"2024-06-16T08:49:27.511183Z","iopub.status.idle":"2024-06-16T08:49:40.053814Z","shell.execute_reply.started":"2024-06-16T08:49:27.511155Z","shell.execute_reply":"2024-06-16T08:49:40.052946Z"},"trusted":true},"execution_count":19,"outputs":[{"name":"stdout","text":"Average Metric: 42 / 50  (84.0): 100%|██████████| 50/50 [00:12<00:00,  3.99it/s]\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"<pandas.io.formats.style.Styler at 0x7deabf1843a0>","text/html":"<style type=\"text/css\">\n#T_65396 th {\n  text-align: left;\n}\n#T_65396 td {\n  text-align: left;\n}\n#T_65396_row0_col0, #T_65396_row0_col1, #T_65396_row0_col2, #T_65396_row0_col3, #T_65396_row1_col0, #T_65396_row1_col1, #T_65396_row1_col2, #T_65396_row1_col3, #T_65396_row2_col0, #T_65396_row2_col1, #T_65396_row2_col2, #T_65396_row2_col3, #T_65396_row3_col0, #T_65396_row3_col1, #T_65396_row3_col2, #T_65396_row3_col3, #T_65396_row4_col0, #T_65396_row4_col1, #T_65396_row4_col2, #T_65396_row4_col3, #T_65396_row5_col0, #T_65396_row5_col1, #T_65396_row5_col2, #T_65396_row5_col3, #T_65396_row6_col0, #T_65396_row6_col1, #T_65396_row6_col2, #T_65396_row6_col3, #T_65396_row7_col0, #T_65396_row7_col1, #T_65396_row7_col2, #T_65396_row7_col3, #T_65396_row8_col0, #T_65396_row8_col1, #T_65396_row8_col2, #T_65396_row8_col3, #T_65396_row9_col0, #T_65396_row9_col1, #T_65396_row9_col2, #T_65396_row9_col3, #T_65396_row10_col0, #T_65396_row10_col1, #T_65396_row10_col2, #T_65396_row10_col3, #T_65396_row11_col0, #T_65396_row11_col1, #T_65396_row11_col2, #T_65396_row11_col3, #T_65396_row12_col0, #T_65396_row12_col1, #T_65396_row12_col2, #T_65396_row12_col3, #T_65396_row13_col0, #T_65396_row13_col1, #T_65396_row13_col2, #T_65396_row13_col3, #T_65396_row14_col0, #T_65396_row14_col1, #T_65396_row14_col2, #T_65396_row14_col3, #T_65396_row15_col0, #T_65396_row15_col1, #T_65396_row15_col2, #T_65396_row15_col3, #T_65396_row16_col0, #T_65396_row16_col1, #T_65396_row16_col2, #T_65396_row16_col3, #T_65396_row17_col0, #T_65396_row17_col1, #T_65396_row17_col2, #T_65396_row17_col3, #T_65396_row18_col0, #T_65396_row18_col1, #T_65396_row18_col2, #T_65396_row18_col3, #T_65396_row19_col0, #T_65396_row19_col1, #T_65396_row19_col2, #T_65396_row19_col3, #T_65396_row20_col0, #T_65396_row20_col1, #T_65396_row20_col2, #T_65396_row20_col3, #T_65396_row21_col0, #T_65396_row21_col1, #T_65396_row21_col2, #T_65396_row21_col3, #T_65396_row22_col0, #T_65396_row22_col1, #T_65396_row22_col2, #T_65396_row22_col3, #T_65396_row23_col0, #T_65396_row23_col1, #T_65396_row23_col2, #T_65396_row23_col3, #T_65396_row24_col0, #T_65396_row24_col1, #T_65396_row24_col2, #T_65396_row24_col3, #T_65396_row25_col0, #T_65396_row25_col1, #T_65396_row25_col2, #T_65396_row25_col3, #T_65396_row26_col0, #T_65396_row26_col1, #T_65396_row26_col2, #T_65396_row26_col3, #T_65396_row27_col0, #T_65396_row27_col1, #T_65396_row27_col2, #T_65396_row27_col3, #T_65396_row28_col0, #T_65396_row28_col1, #T_65396_row28_col2, #T_65396_row28_col3, #T_65396_row29_col0, #T_65396_row29_col1, #T_65396_row29_col2, #T_65396_row29_col3, #T_65396_row30_col0, #T_65396_row30_col1, #T_65396_row30_col2, #T_65396_row30_col3, #T_65396_row31_col0, #T_65396_row31_col1, #T_65396_row31_col2, #T_65396_row31_col3, #T_65396_row32_col0, #T_65396_row32_col1, #T_65396_row32_col2, #T_65396_row32_col3, #T_65396_row33_col0, #T_65396_row33_col1, #T_65396_row33_col2, #T_65396_row33_col3, #T_65396_row34_col0, #T_65396_row34_col1, #T_65396_row34_col2, #T_65396_row34_col3, #T_65396_row35_col0, #T_65396_row35_col1, #T_65396_row35_col2, #T_65396_row35_col3, #T_65396_row36_col0, #T_65396_row36_col1, #T_65396_row36_col2, #T_65396_row36_col3, #T_65396_row37_col0, #T_65396_row37_col1, #T_65396_row37_col2, #T_65396_row37_col3, #T_65396_row38_col0, #T_65396_row38_col1, #T_65396_row38_col2, #T_65396_row38_col3, #T_65396_row39_col0, #T_65396_row39_col1, #T_65396_row39_col2, #T_65396_row39_col3, #T_65396_row40_col0, #T_65396_row40_col1, #T_65396_row40_col2, #T_65396_row40_col3, #T_65396_row41_col0, #T_65396_row41_col1, #T_65396_row41_col2, #T_65396_row41_col3, #T_65396_row42_col0, #T_65396_row42_col1, #T_65396_row42_col2, #T_65396_row42_col3, #T_65396_row43_col0, #T_65396_row43_col1, #T_65396_row43_col2, #T_65396_row43_col3, #T_65396_row44_col0, #T_65396_row44_col1, #T_65396_row44_col2, #T_65396_row44_col3, #T_65396_row45_col0, #T_65396_row45_col1, #T_65396_row45_col2, #T_65396_row45_col3, #T_65396_row46_col0, #T_65396_row46_col1, #T_65396_row46_col2, #T_65396_row46_col3, #T_65396_row47_col0, #T_65396_row47_col1, #T_65396_row47_col2, #T_65396_row47_col3, #T_65396_row48_col0, #T_65396_row48_col1, #T_65396_row48_col2, #T_65396_row48_col3, #T_65396_row49_col0, #T_65396_row49_col1, #T_65396_row49_col2, #T_65396_row49_col3 {\n  text-align: left;\n  white-space: pre-wrap;\n  word-wrap: break-word;\n  max-width: 400px;\n}\n</style>\n<table id=\"T_65396\">\n  <thead>\n    <tr>\n      <th class=\"blank level0\" >&nbsp;</th>\n      <th id=\"T_65396_level0_col0\" class=\"col_heading level0 col0\" >sentence</th>\n      <th id=\"T_65396_level0_col1\" class=\"col_heading level0 col1\" >example_answer</th>\n      <th id=\"T_65396_level0_col2\" class=\"col_heading level0 col2\" >pred_answer</th>\n      <th id=\"T_65396_level0_col3\" class=\"col_heading level0 col3\" >validate_answer</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th id=\"T_65396_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n      <td id=\"T_65396_row0_col0\" class=\"data row0 col0\" >This proved a decisive factor.</td>\n      <td id=\"T_65396_row0_col1\" class=\"data row0 col1\" >1</td>\n      <td id=\"T_65396_row0_col2\" class=\"data row0 col2\" >1</td>\n      <td id=\"T_65396_row0_col3\" class=\"data row0 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n      <td id=\"T_65396_row1_col0\" class=\"data row1 col0\" >he had spent five thousand dollars.</td>\n      <td id=\"T_65396_row1_col1\" class=\"data row1 col1\" >1</td>\n      <td id=\"T_65396_row1_col2\" class=\"data row1 col2\" >1</td>\n      <td id=\"T_65396_row1_col3\" class=\"data row1 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n      <td id=\"T_65396_row2_col0\" class=\"data row2 col0\" >Nora sent Peter the book.</td>\n      <td id=\"T_65396_row2_col1\" class=\"data row2 col1\" >1</td>\n      <td id=\"T_65396_row2_col2\" class=\"data row2 col2\" >1</td>\n      <td id=\"T_65396_row2_col3\" class=\"data row2 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n      <td id=\"T_65396_row3_col0\" class=\"data row3 col0\" >Henry found that Bill is sad.</td>\n      <td id=\"T_65396_row3_col1\" class=\"data row3 col1\" >1</td>\n      <td id=\"T_65396_row3_col2\" class=\"data row3 col2\" >1</td>\n      <td id=\"T_65396_row3_col3\" class=\"data row3 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n      <td id=\"T_65396_row4_col0\" class=\"data row4 col0\" >I found the place where we can relax.</td>\n      <td id=\"T_65396_row4_col1\" class=\"data row4 col1\" >1</td>\n      <td id=\"T_65396_row4_col2\" class=\"data row4 col2\" >1</td>\n      <td id=\"T_65396_row4_col3\" class=\"data row4 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row5\" class=\"row_heading level0 row5\" >5</th>\n      <td id=\"T_65396_row5_col0\" class=\"data row5 col0\" >I know which book Mag read, and which book Bob said that you hadn't.</td>\n      <td id=\"T_65396_row5_col1\" class=\"data row5 col1\" >1</td>\n      <td id=\"T_65396_row5_col2\" class=\"data row5 col2\" >0</td>\n      <td id=\"T_65396_row5_col3\" class=\"data row5 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row6\" class=\"row_heading level0 row6\" >6</th>\n      <td id=\"T_65396_row6_col0\" class=\"data row6 col0\" >Kim alienated cats and beating his dog.</td>\n      <td id=\"T_65396_row6_col1\" class=\"data row6 col1\" >0</td>\n      <td id=\"T_65396_row6_col2\" class=\"data row6 col2\" >0</td>\n      <td id=\"T_65396_row6_col3\" class=\"data row6 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row7\" class=\"row_heading level0 row7\" >7</th>\n      <td id=\"T_65396_row7_col0\" class=\"data row7 col0\" >Most people probably consider, even though the courts didn't actually find, Klaus guilty of murder.</td>\n      <td id=\"T_65396_row7_col1\" class=\"data row7 col1\" >1</td>\n      <td id=\"T_65396_row7_col2\" class=\"data row7 col2\" >1</td>\n      <td id=\"T_65396_row7_col3\" class=\"data row7 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row8\" class=\"row_heading level0 row8\" >8</th>\n      <td id=\"T_65396_row8_col0\" class=\"data row8 col0\" >Came right in he did without so much as a knock.</td>\n      <td id=\"T_65396_row8_col1\" class=\"data row8 col1\" >1</td>\n      <td id=\"T_65396_row8_col2\" class=\"data row8 col2\" >0</td>\n      <td id=\"T_65396_row8_col3\" class=\"data row8 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row9\" class=\"row_heading level0 row9\" >9</th>\n      <td id=\"T_65396_row9_col0\" class=\"data row9 col0\" >We pulled free.</td>\n      <td id=\"T_65396_row9_col1\" class=\"data row9 col1\" >1</td>\n      <td id=\"T_65396_row9_col2\" class=\"data row9 col2\" >1</td>\n      <td id=\"T_65396_row9_col3\" class=\"data row9 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row10\" class=\"row_heading level0 row10\" >10</th>\n      <td id=\"T_65396_row10_col0\" class=\"data row10 col0\" >Sarah promised Catherine her old car, but then gave it to her son instead.</td>\n      <td id=\"T_65396_row10_col1\" class=\"data row10 col1\" >1</td>\n      <td id=\"T_65396_row10_col2\" class=\"data row10 col2\" >1</td>\n      <td id=\"T_65396_row10_col3\" class=\"data row10 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row11\" class=\"row_heading level0 row11\" >11</th>\n      <td id=\"T_65396_row11_col0\" class=\"data row11 col0\" >After reading the pamphlet, Judy threw it into the garbage can.</td>\n      <td id=\"T_65396_row11_col1\" class=\"data row11 col1\" >1</td>\n      <td id=\"T_65396_row11_col2\" class=\"data row11 col2\" >1</td>\n      <td id=\"T_65396_row11_col3\" class=\"data row11 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row12\" class=\"row_heading level0 row12\" >12</th>\n      <td id=\"T_65396_row12_col0\" class=\"data row12 col0\" >Kim put in the box.</td>\n      <td id=\"T_65396_row12_col1\" class=\"data row12 col1\" >0</td>\n      <td id=\"T_65396_row12_col2\" class=\"data row12 col2\" >1</td>\n      <td id=\"T_65396_row12_col3\" class=\"data row12 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row13\" class=\"row_heading level0 row13\" >13</th>\n      <td id=\"T_65396_row13_col0\" class=\"data row13 col0\" >John tried to please Stephen.</td>\n      <td id=\"T_65396_row13_col1\" class=\"data row13 col1\" >1</td>\n      <td id=\"T_65396_row13_col2\" class=\"data row13 col2\" >1</td>\n      <td id=\"T_65396_row13_col3\" class=\"data row13 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row14\" class=\"row_heading level0 row14\" >14</th>\n      <td id=\"T_65396_row14_col0\" class=\"data row14 col0\" >It was the peasant girl who got it.</td>\n      <td id=\"T_65396_row14_col1\" class=\"data row14 col1\" >1</td>\n      <td id=\"T_65396_row14_col2\" class=\"data row14 col2\" >1</td>\n      <td id=\"T_65396_row14_col3\" class=\"data row14 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row15\" class=\"row_heading level0 row15\" >15</th>\n      <td id=\"T_65396_row15_col0\" class=\"data row15 col0\" >Anson believed himself to be handsome.</td>\n      <td id=\"T_65396_row15_col1\" class=\"data row15 col1\" >1</td>\n      <td id=\"T_65396_row15_col2\" class=\"data row15 col2\" >1</td>\n      <td id=\"T_65396_row15_col3\" class=\"data row15 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row16\" class=\"row_heading level0 row16\" >16</th>\n      <td id=\"T_65396_row16_col0\" class=\"data row16 col0\" >This girl in the red coat will put a picture of Bill in the mailbox and on your desk before tomorrow.</td>\n      <td id=\"T_65396_row16_col1\" class=\"data row16 col1\" >1</td>\n      <td id=\"T_65396_row16_col2\" class=\"data row16 col2\" >1</td>\n      <td id=\"T_65396_row16_col3\" class=\"data row16 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row17\" class=\"row_heading level0 row17\" >17</th>\n      <td id=\"T_65396_row17_col0\" class=\"data row17 col0\" >Mike talked about politics yesterday to my friends.</td>\n      <td id=\"T_65396_row17_col1\" class=\"data row17 col1\" >1</td>\n      <td id=\"T_65396_row17_col2\" class=\"data row17 col2\" >1</td>\n      <td id=\"T_65396_row17_col3\" class=\"data row17 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row18\" class=\"row_heading level0 row18\" >18</th>\n      <td id=\"T_65396_row18_col0\" class=\"data row18 col0\" >John is eager.</td>\n      <td id=\"T_65396_row18_col1\" class=\"data row18 col1\" >1</td>\n      <td id=\"T_65396_row18_col2\" class=\"data row18 col2\" >1</td>\n      <td id=\"T_65396_row18_col3\" class=\"data row18 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row19\" class=\"row_heading level0 row19\" >19</th>\n      <td id=\"T_65396_row19_col0\" class=\"data row19 col0\" >Any owl hunts mice.</td>\n      <td id=\"T_65396_row19_col1\" class=\"data row19 col1\" >1</td>\n      <td id=\"T_65396_row19_col2\" class=\"data row19 col2\" >1</td>\n      <td id=\"T_65396_row19_col3\" class=\"data row19 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row20\" class=\"row_heading level0 row20\" >20</th>\n      <td id=\"T_65396_row20_col0\" class=\"data row20 col0\" >The car was driven.</td>\n      <td id=\"T_65396_row20_col1\" class=\"data row20 col1\" >1</td>\n      <td id=\"T_65396_row20_col2\" class=\"data row20 col2\" >1</td>\n      <td id=\"T_65396_row20_col3\" class=\"data row20 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row21\" class=\"row_heading level0 row21\" >21</th>\n      <td id=\"T_65396_row21_col0\" class=\"data row21 col0\" >We linguists love to argue</td>\n      <td id=\"T_65396_row21_col1\" class=\"data row21 col1\" >1</td>\n      <td id=\"T_65396_row21_col2\" class=\"data row21 col2\" >1</td>\n      <td id=\"T_65396_row21_col3\" class=\"data row21 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row22\" class=\"row_heading level0 row22\" >22</th>\n      <td id=\"T_65396_row22_col0\" class=\"data row22 col0\" >John often meets Mary.</td>\n      <td id=\"T_65396_row22_col1\" class=\"data row22 col1\" >1</td>\n      <td id=\"T_65396_row22_col2\" class=\"data row22 col2\" >1</td>\n      <td id=\"T_65396_row22_col3\" class=\"data row22 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row23\" class=\"row_heading level0 row23\" >23</th>\n      <td id=\"T_65396_row23_col0\" class=\"data row23 col0\" >The farmer loaded the cart with apples.</td>\n      <td id=\"T_65396_row23_col1\" class=\"data row23 col1\" >1</td>\n      <td id=\"T_65396_row23_col2\" class=\"data row23 col2\" >1</td>\n      <td id=\"T_65396_row23_col3\" class=\"data row23 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row24\" class=\"row_heading level0 row24\" >24</th>\n      <td id=\"T_65396_row24_col0\" class=\"data row24 col0\" >The government's imposition of a fine.</td>\n      <td id=\"T_65396_row24_col1\" class=\"data row24 col1\" >1</td>\n      <td id=\"T_65396_row24_col2\" class=\"data row24 col2\" >0</td>\n      <td id=\"T_65396_row24_col3\" class=\"data row24 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row25\" class=\"row_heading level0 row25\" >25</th>\n      <td id=\"T_65396_row25_col0\" class=\"data row25 col0\" >Mickey looked up it.</td>\n      <td id=\"T_65396_row25_col1\" class=\"data row25 col1\" >0</td>\n      <td id=\"T_65396_row25_col2\" class=\"data row25 col2\" >0</td>\n      <td id=\"T_65396_row25_col3\" class=\"data row25 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row26\" class=\"row_heading level0 row26\" >26</th>\n      <td id=\"T_65396_row26_col0\" class=\"data row26 col0\" >No writer, and no playwright, meets in Vienna.</td>\n      <td id=\"T_65396_row26_col1\" class=\"data row26 col1\" >0</td>\n      <td id=\"T_65396_row26_col2\" class=\"data row26 col2\" >1</td>\n      <td id=\"T_65396_row26_col3\" class=\"data row26 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row27\" class=\"row_heading level0 row27\" >27</th>\n      <td id=\"T_65396_row27_col0\" class=\"data row27 col0\" >Frank Churchill crossed the street.</td>\n      <td id=\"T_65396_row27_col1\" class=\"data row27 col1\" >1</td>\n      <td id=\"T_65396_row27_col2\" class=\"data row27 col2\" >1</td>\n      <td id=\"T_65396_row27_col3\" class=\"data row27 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row28\" class=\"row_heading level0 row28\" >28</th>\n      <td id=\"T_65396_row28_col0\" class=\"data row28 col0\" >Suddenly, there arrived two inspectors from the INS.</td>\n      <td id=\"T_65396_row28_col1\" class=\"data row28 col1\" >1</td>\n      <td id=\"T_65396_row28_col2\" class=\"data row28 col2\" >1</td>\n      <td id=\"T_65396_row28_col3\" class=\"data row28 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row29\" class=\"row_heading level0 row29\" >29</th>\n      <td id=\"T_65396_row29_col0\" class=\"data row29 col0\" >I'm sure we even got these tickets!</td>\n      <td id=\"T_65396_row29_col1\" class=\"data row29 col1\" >1</td>\n      <td id=\"T_65396_row29_col2\" class=\"data row29 col2\" >1</td>\n      <td id=\"T_65396_row29_col3\" class=\"data row29 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row30\" class=\"row_heading level0 row30\" >30</th>\n      <td id=\"T_65396_row30_col0\" class=\"data row30 col0\" >One of Korea's most famous poets wrote these lines.</td>\n      <td id=\"T_65396_row30_col1\" class=\"data row30 col1\" >1</td>\n      <td id=\"T_65396_row30_col2\" class=\"data row30 col2\" >1</td>\n      <td id=\"T_65396_row30_col3\" class=\"data row30 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row31\" class=\"row_heading level0 row31\" >31</th>\n      <td id=\"T_65396_row31_col0\" class=\"data row31 col0\" >Heidi thinks that Andy should eat salmon flavored candy bars.</td>\n      <td id=\"T_65396_row31_col1\" class=\"data row31 col1\" >1</td>\n      <td id=\"T_65396_row31_col2\" class=\"data row31 col2\" >1</td>\n      <td id=\"T_65396_row31_col3\" class=\"data row31 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row32\" class=\"row_heading level0 row32\" >32</th>\n      <td id=\"T_65396_row32_col0\" class=\"data row32 col0\" >The box contained the ball from the tree.</td>\n      <td id=\"T_65396_row32_col1\" class=\"data row32 col1\" >0</td>\n      <td id=\"T_65396_row32_col2\" class=\"data row32 col2\" >1</td>\n      <td id=\"T_65396_row32_col3\" class=\"data row32 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row33\" class=\"row_heading level0 row33\" >33</th>\n      <td id=\"T_65396_row33_col0\" class=\"data row33 col0\" >A job offered.</td>\n      <td id=\"T_65396_row33_col1\" class=\"data row33 col1\" >0</td>\n      <td id=\"T_65396_row33_col2\" class=\"data row33 col2\" >0</td>\n      <td id=\"T_65396_row33_col3\" class=\"data row33 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row34\" class=\"row_heading level0 row34\" >34</th>\n      <td id=\"T_65396_row34_col0\" class=\"data row34 col0\" >I want to try and buy some whiskey.</td>\n      <td id=\"T_65396_row34_col1\" class=\"data row34 col1\" >1</td>\n      <td id=\"T_65396_row34_col2\" class=\"data row34 col2\" >1</td>\n      <td id=\"T_65396_row34_col3\" class=\"data row34 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row35\" class=\"row_heading level0 row35\" >35</th>\n      <td id=\"T_65396_row35_col0\" class=\"data row35 col0\" >Jackie accompanied Rose.</td>\n      <td id=\"T_65396_row35_col1\" class=\"data row35 col1\" >1</td>\n      <td id=\"T_65396_row35_col2\" class=\"data row35 col2\" >1</td>\n      <td id=\"T_65396_row35_col3\" class=\"data row35 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row36\" class=\"row_heading level0 row36\" >36</th>\n      <td id=\"T_65396_row36_col0\" class=\"data row36 col0\" >Every student, and he wears socks, is a swinger.</td>\n      <td id=\"T_65396_row36_col1\" class=\"data row36 col1\" >0</td>\n      <td id=\"T_65396_row36_col2\" class=\"data row36 col2\" >0</td>\n      <td id=\"T_65396_row36_col3\" class=\"data row36 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row37\" class=\"row_heading level0 row37\" >37</th>\n      <td id=\"T_65396_row37_col0\" class=\"data row37 col0\" >That the fuzz wanted him worried John, but that the fuzz wanted her didn't worry Mary.</td>\n      <td id=\"T_65396_row37_col1\" class=\"data row37 col1\" >1</td>\n      <td id=\"T_65396_row37_col2\" class=\"data row37 col2\" >0</td>\n      <td id=\"T_65396_row37_col3\" class=\"data row37 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row38\" class=\"row_heading level0 row38\" >38</th>\n      <td id=\"T_65396_row38_col0\" class=\"data row38 col0\" >Drowning cats, which is against the law, are hard to rescue.</td>\n      <td id=\"T_65396_row38_col1\" class=\"data row38 col1\" >0</td>\n      <td id=\"T_65396_row38_col2\" class=\"data row38 col2\" >0</td>\n      <td id=\"T_65396_row38_col3\" class=\"data row38 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row39\" class=\"row_heading level0 row39\" >39</th>\n      <td id=\"T_65396_row39_col0\" class=\"data row39 col0\" >We believed John to be a fountain in the park.</td>\n      <td id=\"T_65396_row39_col1\" class=\"data row39 col1\" >0</td>\n      <td id=\"T_65396_row39_col2\" class=\"data row39 col2\" >0</td>\n      <td id=\"T_65396_row39_col3\" class=\"data row39 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row40\" class=\"row_heading level0 row40\" >40</th>\n      <td id=\"T_65396_row40_col0\" class=\"data row40 col0\" >This flyer and that flyer differ.</td>\n      <td id=\"T_65396_row40_col1\" class=\"data row40 col1\" >1</td>\n      <td id=\"T_65396_row40_col2\" class=\"data row40 col2\" >1</td>\n      <td id=\"T_65396_row40_col3\" class=\"data row40 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row41\" class=\"row_heading level0 row41\" >41</th>\n      <td id=\"T_65396_row41_col0\" class=\"data row41 col0\" >The man I saw get into the cab robbed the bank.</td>\n      <td id=\"T_65396_row41_col1\" class=\"data row41 col1\" >1</td>\n      <td id=\"T_65396_row41_col2\" class=\"data row41 col2\" >1</td>\n      <td id=\"T_65396_row41_col3\" class=\"data row41 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row42\" class=\"row_heading level0 row42\" >42</th>\n      <td id=\"T_65396_row42_col0\" class=\"data row42 col0\" >This teacher is a genius.</td>\n      <td id=\"T_65396_row42_col1\" class=\"data row42 col1\" >1</td>\n      <td id=\"T_65396_row42_col2\" class=\"data row42 col2\" >1</td>\n      <td id=\"T_65396_row42_col3\" class=\"data row42 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row43\" class=\"row_heading level0 row43\" >43</th>\n      <td id=\"T_65396_row43_col0\" class=\"data row43 col0\" >Linda taped the picture to the wall.</td>\n      <td id=\"T_65396_row43_col1\" class=\"data row43 col1\" >1</td>\n      <td id=\"T_65396_row43_col2\" class=\"data row43 col2\" >1</td>\n      <td id=\"T_65396_row43_col3\" class=\"data row43 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row44\" class=\"row_heading level0 row44\" >44</th>\n      <td id=\"T_65396_row44_col0\" class=\"data row44 col0\" >I live at the place where Route 150 crosses the Hudson River and my dad lives at it too.</td>\n      <td id=\"T_65396_row44_col1\" class=\"data row44 col1\" >0</td>\n      <td id=\"T_65396_row44_col2\" class=\"data row44 col2\" >1</td>\n      <td id=\"T_65396_row44_col3\" class=\"data row44 col3\" >False</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row45\" class=\"row_heading level0 row45\" >45</th>\n      <td id=\"T_65396_row45_col0\" class=\"data row45 col0\" >Have you seen my model airplane collection?</td>\n      <td id=\"T_65396_row45_col1\" class=\"data row45 col1\" >1</td>\n      <td id=\"T_65396_row45_col2\" class=\"data row45 col2\" >1</td>\n      <td id=\"T_65396_row45_col3\" class=\"data row45 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row46\" class=\"row_heading level0 row46\" >46</th>\n      <td id=\"T_65396_row46_col0\" class=\"data row46 col0\" >What did you ask who Medea gave?</td>\n      <td id=\"T_65396_row46_col1\" class=\"data row46 col1\" >0</td>\n      <td id=\"T_65396_row46_col2\" class=\"data row46 col2\" >0</td>\n      <td id=\"T_65396_row46_col3\" class=\"data row46 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row47\" class=\"row_heading level0 row47\" >47</th>\n      <td id=\"T_65396_row47_col0\" class=\"data row47 col0\" >Who did John send the book?</td>\n      <td id=\"T_65396_row47_col1\" class=\"data row47 col1\" >1</td>\n      <td id=\"T_65396_row47_col2\" class=\"data row47 col2\" >1</td>\n      <td id=\"T_65396_row47_col3\" class=\"data row47 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row48\" class=\"row_heading level0 row48\" >48</th>\n      <td id=\"T_65396_row48_col0\" class=\"data row48 col0\" >Carla shoveled the walk.</td>\n      <td id=\"T_65396_row48_col1\" class=\"data row48 col1\" >1</td>\n      <td id=\"T_65396_row48_col2\" class=\"data row48 col2\" >1</td>\n      <td id=\"T_65396_row48_col3\" class=\"data row48 col3\" >✔️ [True]</td>\n    </tr>\n    <tr>\n      <th id=\"T_65396_level0_row49\" class=\"row_heading level0 row49\" >49</th>\n      <td id=\"T_65396_row49_col0\" class=\"data row49 col0\" >I know you eat asparagus.</td>\n      <td id=\"T_65396_row49_col1\" class=\"data row49 col1\" >1</td>\n      <td id=\"T_65396_row49_col2\" class=\"data row49 col2\" >1</td>\n      <td id=\"T_65396_row49_col3\" class=\"data row49 col3\" >✔️ [True]</td>\n    </tr>\n  </tbody>\n</table>\n"},"metadata":{}},{"execution_count":19,"output_type":"execute_result","data":{"text/plain":"84.0"},"metadata":{}}]},{"cell_type":"code","source":"lm.history[-1]['prompt']","metadata":{"execution":{"iopub.status.busy":"2024-06-16T08:50:48.150939Z","iopub.execute_input":"2024-06-16T08:50:48.151879Z","iopub.status.idle":"2024-06-16T08:50:48.157845Z","shell.execute_reply.started":"2024-06-16T08:50:48.151845Z","shell.execute_reply":"2024-06-16T08:50:48.156839Z"},"trusted":true},"execution_count":21,"outputs":[{"execution_count":21,"output_type":"execute_result","data":{"text/plain":"\"Determine if the given sentence adheres to standard English grammar rules by outputting a '1' for grammatically correct and '0' for incorrect.\\n\\n---\\n\\nFollow the following format.\\n\\nSentence: a sentence in English\\nIf the input sentence is grammatically correct, please respond with 1; otherwise, respond with 0. predicted label only (1 or 0 only)\\n\\n---\\n\\nSentence: I know you eat asparagus.\\nIf the input sentence is grammatically correct, please respond with 1; otherwise, respond with 0.\""},"metadata":{}}]},{"cell_type":"markdown","source":"<h3>much better!</h3>","metadata":{}},{"cell_type":"code","source":"lm.history[-1]['prompt']","metadata":{"execution":{"iopub.status.busy":"2024-06-16T09:14:08.511546Z","iopub.execute_input":"2024-06-16T09:14:08.512261Z","iopub.status.idle":"2024-06-16T09:14:08.518008Z","shell.execute_reply.started":"2024-06-16T09:14:08.512229Z","shell.execute_reply":"2024-06-16T09:14:08.517097Z"},"trusted":true},"execution_count":37,"outputs":[{"execution_count":37,"output_type":"execute_result","data":{"text/plain":"\"Determine if the given sentence adheres to standard English grammar rules by outputting a '1' for grammatically correct and '0' for incorrect.\\n\\n---\\n\\nFollow the following format.\\n\\nSentence: a sentence in English\\nIf the input sentence is grammatically correct, please respond with 1; otherwise, respond with 0. predicted label only (1 or 0 only)\\n\\n---\\n\\nSentence: I know you eat asparagus.\\nIf the input sentence is grammatically correct, please respond with 1; otherwise, respond with 0.\""},"metadata":{}}]},{"cell_type":"code","source":"dataset[\"test\"]","metadata":{"execution":{"iopub.status.busy":"2024-06-16T09:15:30.151664Z","iopub.execute_input":"2024-06-16T09:15:30.152637Z","iopub.status.idle":"2024-06-16T09:15:30.158332Z","shell.execute_reply.started":"2024-06-16T09:15:30.152601Z","shell.execute_reply":"2024-06-16T09:15:30.157556Z"},"trusted":true},"execution_count":42,"outputs":[{"execution_count":42,"output_type":"execute_result","data":{"text/plain":"Dataset({\n    features: ['text', 'label'],\n    num_rows: 527\n})"},"metadata":{}}]}]}