new_data_generation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import os\n",
    "import re\n",
    "import yaml\n",
    "import math\n",
    "import warnings\n",
    "from openai import OpenAI\n",
    "from groq import Groq\n",
    "from tqdm import tqdm\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "config_path = os.path.join(os.getcwd(), \"config.yaml\")\n",
    "\n",
    "with open(config_path, \"r\") as file:\n",
    "    config = yaml.safe_load(file)\n",
    "\n",
    "openai_api_key = config[\"openai\"][\"api_key\"]\n",
    "openai_organization = config[\"openai\"][\"organization\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "client_openai = OpenAI(api_key=openai_api_key, organization=openai_organization)\n",
    "\n",
    "def get_response_openai(model, prompt, temperature=0.7, top_p=0.95, max_tokens=2048):\n",
    "    completion = client_openai.chat.completions.create(\n",
    "        model=model,\n",
    "        messages=[\n",
    "            {\"role\": \"user\", \"content\": prompt}\n",
    "        ],\n",
    "        temperature=temperature,\n",
    "        top_p=top_p,\n",
    "        max_tokens=max_tokens\n",
    "    )\n",
    "    response = completion.choices[0].message.content\n",
    "    return response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "misconception_mapping = pd.read_csv(\"./comp_data/misconception_mapping.csv\")\n",
    "print(misconception_mapping.shape)\n",
    "misconception_mapping.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = \"\"\"You are tasked with designing a complex math-related question, its correct answer, and an incorrect answer that demonstrates a specific type of mistake. Follow these instructions carefully:\n",
    "\n",
    "1. Create a complex math question suitable for high school or early college-level students. The question should be challenging but solvable with standard mathematical knowledge.\n",
    "\n",
    "2. Determine the correct answer to your question. Generate your solution process in <correct_solution> tags, your solution must be correct, clear and logical.\n",
    "\n",
    "3. Generate an incorrect answer that demonstrates the following type of mistake:\n",
    "<mistake_type>\n",
    "{{MISTAKE_TYPE}}\n",
    "</mistake_type>\n",
    "\n",
    "4. Format your output as follows:\n",
    "   - Enclose the question within <question> tags\n",
    "   - Enclose the correct answer within <correct_answer> tags (only the correct answer, do not contain the solution process)\n",
    "   - Enclose the incorrect answer within <incorrect_answer> tags (only the incorrect answer)\n",
    "   - Include a brief explanation in 50 words of how the incorrect answer demonstrates the specified mistake type within <explanation> tags\n",
    "\n",
    "Remember to make the question sufficiently complex and ensure that the incorrect answer clearly demonstrates the specified mistake type. Be creative in your question design while maintaining mathematical accuracy and relevance.\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_data = pd.DataFrame(columns=[\"QuestionText\", \"CorrectAnswer\", \"IncorrectAnswer\", \"MisconceptionId\", \"MisconceptionName\", \"UsedModel\", \"input_prompt\"])\n",
    "new_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "misconception_mapping = misconception_mapping[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx, row in tqdm(misconception_mapping.iterrows(), total=len(misconception_mapping)):\n",
    "    mistake_type = row[\"MisconceptionName\"]\n",
    "    input_prompt = prompt.replace(\"{{MISTAKE_TYPE}}\", mistake_type)\n",
    "    try:\n",
    "        for (model, temperature, top_p) in [(\"gpt-4o-mini\", 0.0, 0.95)]:\n",
    "            response = get_response_openai(\n",
    "                model=model, \n",
    "                prompt=input_prompt, \n",
    "                temperature=temperature, \n",
    "                top_p=top_p\n",
    "            )\n",
    "            # print(response)\n",
    "            question = re.search(r\"<question>(.*?)</question>\", response, re.DOTALL).group(1).strip()\n",
    "            correct_answer = re.search(r\"<correct_answer>(.*?)</correct_answer>\", response, re.DOTALL).group(1).strip()\n",
    "            incorrect_answer = re.search(r\"<incorrect_answer>(.*?)</incorrect_answer>\", response, re.DOTALL).group(1).strip()\n",
    "            new_data_row = pd.DataFrame([{\n",
    "                \"QuestionText\": question,\n",
    "                \"CorrectAnswer\": correct_answer,\n",
    "                \"IncorrectAnswer\": incorrect_answer,\n",
    "                \"MisconceptionId\": row[\"MisconceptionId\"],\n",
    "                \"MisconceptionName\": row[\"MisconceptionName\"],\n",
    "                \"UsedModel\": model,\n",
    "                \"input_prompt\": input_prompt\n",
    "            }])\n",
    "            new_data = pd.concat([new_data, new_data_row], ignore_index=True)\n",
    "    except Exception as e:\n",
    "        print(f\"Error for misconception {row['MisconceptionName']}: {e}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(new_data[\"QuestionText\"].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_data.to_csv(\"./adv_data/new_generated_data.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}