SAQUET.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 337,
   "id": "b3658995-00c0-4902-a90e-70b9d97d9a61",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\Steven\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\Steven\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package punkt_tab to\n",
      "[nltk_data]     C:\\Users\\Steven\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt_tab is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger_eng to\n",
      "[nltk_data]     C:\\Users\\Steven\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger_eng is already up-to-\n",
      "[nltk_data]       date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     C:\\Users\\Steven\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\Steven\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "#Import the required libraries, lots of these are required for the LLMs we utilize for three criteria. \n",
    "import Levenshtein\n",
    "import spacy\n",
    "import string\n",
    "import pandas as pd\n",
    "from transformers import pipeline, AutoTokenizer, AutoModel, AutoModelForSequenceClassification, GPT2LMHeadModel, GPT2TokenizerFast, OpenAIGPTTokenizer, OpenAIGPTLMHeadModel\n",
    "import torch\n",
    "from sentence_transformers import SentenceTransformer, util\n",
    "from lexicalrichness import LexicalRichness\n",
    "import re\n",
    "import math\n",
    "import random\n",
    "import numpy as np\n",
    "nlp = spacy.load('en_core_web_lg')\n",
    "import time\n",
    "from datetime import datetime\n",
    "from collections import Counter\n",
    "import warnings\n",
    "import json\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "#NLTK Imports\n",
    "import nltk\n",
    "nltk.download('stopwords') #Needed for query wellformedness\n",
    "nltk.download('punkt') #Needed for query wellformedness\n",
    "nltk.download('punkt_tab')\n",
    "nltk.download('averaged_perceptron_tagger_eng')\n",
    "nltk.download('averaged_perceptron_tagger') #Needed for query wellformedness\n",
    "nltk.download('wordnet')\n",
    "from nltk.corpus import wordnet as wn\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.corpus import stopwords\n",
    "from nltk import trigrams\n",
    "from nltk import ngrams\n",
    "stop_words = set(stopwords.words('english'))\n",
    "from nltk.tokenize import word_tokenize, sent_tokenize\n",
    "\n",
    "#OpenAI, but could be replaced with Gemini, Claude, etc.\n",
    "from openai import OpenAI\n",
    "model_engine = 'gpt-4o' #or the other models 'gpt-4o-mini','chatgpt-4o-latest', 'o1-preview' etc.\n",
    "client = OpenAI(\n",
    "    api_key='OPENAI_API_KEY', \n",
    ")\n",
    "\n",
    "#Libraries for Perplexity, Diversity, Grammatical Error, Complexity, Answerability\n",
    "from evaluate import load\n",
    "import language_tool_python"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1243aa51-9182-464e-8168-90875e536870",
   "metadata": {},
   "source": [
    "## Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "id": "34f13afe-dd87-4abb-a567-7c9ec319d341",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Our class used to represent multiple-choice questions\n",
    "nl = '\\n'\n",
    "class MultipleChoiceQuestion:\n",
    "    def __init__(self, stem, options, correct_option, qid = None, quality = None):\n",
    "        self.stem = stem\n",
    "        self.options = options\n",
    "        self.correct_option = correct_option\n",
    "        self.qid = qid\n",
    "        self.quality = quality\n",
    "        \n",
    "    def __str__(self):\n",
    "        return f\"Question: {self.stem}\\n {nl.join(self.options)}\\nCorrect option: {self.correct_option}\\nQuality: {self.quality}\"\n",
    "\n",
    "def mode_or_average(lst):\n",
    "    # Count occurrences of each number\n",
    "    try:\n",
    "        lst = [float(x) for x in lst]  # Handles strings of numbers\n",
    "    except ValueError:\n",
    "        raise ValueError(\"All elements in the list must be convertible to numbers.\")\n",
    "        \n",
    "    count = Counter(lst)\n",
    "    \n",
    "    # Find the maximum frequency\n",
    "    max_freq = max(count.values())\n",
    "    # Find all numbers with the maximum frequency\n",
    "    modes = [k for k, v in count.items() if v == max_freq]\n",
    "\n",
    "    if len(modes) == 1:\n",
    "        # If there's a single mode, return it\n",
    "        return modes[0]\n",
    "    else:\n",
    "        # If no single mode, compute the average of two closest numbers\n",
    "        sorted_lst = sorted(lst)\n",
    "        min_diff = float('inf')\n",
    "        closest_pair = None\n",
    "\n",
    "        # Find the closest pair\n",
    "        for i in range(len(sorted_lst) - 1):\n",
    "            diff = sorted_lst[i + 1] - sorted_lst[i]\n",
    "            if diff < min_diff:\n",
    "                min_diff = diff\n",
    "                closest_pair = (sorted_lst[i], sorted_lst[i + 1])\n",
    "        \n",
    "        # Compute and return the average of the closest pair\n",
    "        return np.mean(closest_pair)\n",
    "\n",
    "#Rating-based LLM Logic used for all verification steps where the LLM is needed\n",
    "def llm_rating(sysrole, prompt, threshold=5):\n",
    "    done = False\n",
    "    ratings = []\n",
    "    while not done:\n",
    "        try:\n",
    "            o = client.chat.completions.create(\n",
    "              model=model_engine,\n",
    "              messages=[\n",
    "                 {\"role\": \"system\",\n",
    "                  \"content\": sysrole},\n",
    "                {\"role\": \"user\", \n",
    "                 \"content\": prompt},\n",
    "              ],\n",
    "              max_tokens = 4096,\n",
    "              temperature = 0.7\n",
    "             )\n",
    "            result = o.choices[0].message.content.lower().strip()\n",
    "            if result.isdigit():\n",
    "                ratings.append(result)\n",
    "        except Exception as error:\n",
    "            print('errored in LLM API call: ', error)\n",
    "            time.sleep(10)\n",
    "        if len(ratings) == 3:\n",
    "            done = True\n",
    "        if len(ratings) == 2:\n",
    "            if float(ratings[0]) >= 7 and float(ratings[1]) >= 7:\n",
    "                done = True\n",
    "            if float(ratings[0]) <= 4 and float(ratings[1]) <= 4:\n",
    "                done = True\n",
    "                \n",
    "    rating = mode_or_average(ratings)\n",
    "    \n",
    "    return rating >= threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "581b3cd5-19c2-4af5-b973-57448e8f9902",
   "metadata": {},
   "source": [
    "## Implausible Distractors\n",
    "Make all distractors plausible as good items depend on having effective distractors\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "id": "ee242951-010a-4fc7-aa24-a973ad3fd260",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Uses NER, so if the score is too low, if they're matching entities (i.e. people) then we can ignore this case and say True\n",
    "def implausible_distractors(question):\n",
    "    #MiniLM from: https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2\n",
    "    model = SentenceTransformer('all-MiniLM-L6-v2')\n",
    "    correct = question.correct_option\n",
    "    options = question.options.copy()\n",
    "\n",
    "    # Split the correct answers (if multiple) and remove them from options\n",
    "    correct_answers = [opt.strip() for opt in question.correct_option.split('[SEP]')]\n",
    "    for opt in correct_answers:\n",
    "        if opt in options:\n",
    "            options.remove(opt)\n",
    "            \n",
    "    #Compute embedding for both lists\n",
    "    embeddings1 = model.encode([correct] * len(options), convert_to_tensor=True)\n",
    "    embeddings2 = model.encode(options, convert_to_tensor=True)\n",
    "    \n",
    "    #Compute cosine-similarities\n",
    "    cosine_scores = util.cos_sim(embeddings1, embeddings2)\n",
    "\n",
    "    #Output the pairs with their score\n",
    "    for i, opt in enumerate(options):\n",
    "        similarity_score = cosine_scores[i][i].item()\n",
    "\n",
    "        if similarity_score < 0.15:\n",
    "            # Perform Named Entity Recognition (NER) checks\n",
    "            opt_entity = nlp(opt)\n",
    "            lemma_nouns_opt = get_lemma_nouns(opt)\n",
    "            \n",
    "            ans_entity = nlp(correct)\n",
    "            lemma_nouns_ans = get_lemma_nouns(correct)\n",
    "\n",
    "            opt_entity_label = opt_entity.ents[0].label_ if opt_entity.ents else None\n",
    "            ans_entity_label = ans_entity.ents[0].label_ if ans_entity.ents else None\n",
    "\n",
    "            # If no entities or nouns can be parsed, we assume it's safe\n",
    "            if not lemma_nouns_ans and not lemma_nouns_opt:\n",
    "                return True\n",
    "\n",
    "            # If it’s an \"All of the above\" or \"None of the above\" case, ignore similarity check\n",
    "            if not all_of_the_above(question) or not none_of_the_above(question):\n",
    "                return True\n",
    "\n",
    "            # Apply Jaccard similarity and Levenshtein distance rules\n",
    "            if jaccard_similarity(correct, opt) > 0.15 or Levenshtein.distance(correct, opt) < (len(correct) * 0.7):\n",
    "                return True\n",
    "\n",
    "            # LLM-based final check\n",
    "            if implausible_distractors_verify(question):\n",
    "                print('LLM confirms similarity: ', correct, ' -and- ', opt)\n",
    "                return True\n",
    "\n",
    "            print(f\"Distractor too different: '{correct}' vs. '{opt}' | Score: {similarity_score:.4f}\")\n",
    "            return False\n",
    "\n",
    "    return True\n",
    "\n",
    "#Statistic used for gauging the similarity and diversity of text\n",
    "def jaccard_similarity(str1, str2):\n",
    "    # Convert strings to sets of words\n",
    "    set1 = set(str1.split())\n",
    "    set2 = set(str2.split())\n",
    "\n",
    "    # Calculate intersection and union\n",
    "    intersection = set1.intersection(set2)\n",
    "    union = set1.union(set2)\n",
    "\n",
    "    # Calculate Jaccard Similarity\n",
    "    similarity = len(intersection) / len(union)\n",
    "    return similarity\n",
    "\n",
    "def implausible_distractors_verify(question):\n",
    "    sysrole = \"\"\"You are a seasoned academic professional with extensive experience in designing and reviewing multiple-choice assessments. Your primary objective is to identify whether any distractors (incorrect answer choices) in a multiple-choice question are overly implausible—so unrelated or off-topic that students, even those with minimal subject knowledge, would not consider selecting them. Your assessment should focus on how well each distractor aligns with common misconceptions, partial knowledge, or thematic similarity to the correct answer.\"\"\"\n",
    "    prompt = \"\"\"\n",
    "Instructions:\n",
    "    Assign a single numeric score from 1 to 10 indicating the plausibility of the distractors:\n",
    "\n",
    "    Scoring Guidelines:\n",
    "        1–3: At least one distractor is clearly off-topic, illogical, or so unrelated to the question, such that no student, even with minimal subject knowledge, would choose it.\n",
    "        4–6: Distractors have noticeable plausibility issues; there may be some relevance, but at least one distractor still seems very out of place or unconnected to the question or other options.\n",
    "        7–9: Distractors are reasonably plausible and relevant, though there might be minor clues or inconsistencies that reduce their effectiveness.\n",
    "        10: All distractors are reasonably related or plausible, so a student with limited knowledge might actually pick any of them due to confusion, misunderstanding, or similarity to the correct answer.\n",
    "        \n",
    "    Respond with only the numeric score (1–10) and nothing else. Do not provide an explanation or any additional commentary.\n",
    "\n",
    "Question: {}\n",
    "Options: {}\n",
    "Answer: {}\"\"\".format(question.stem, question.options, question.correct_option)\n",
    "    \n",
    "    return llm_rating(sysrole, prompt)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a56569c-84cc-41ba-8077-c54f508d4487",
   "metadata": {},
   "source": [
    "## None Of The Above\n",
    "Avoid none of the above as it only really measures students ability to detect incorrect answers "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "id": "cd69171a-423c-4dee-a69a-c977b9850e07",
   "metadata": {},
   "outputs": [],
   "source": [
    "def none_of_the_above(question):\n",
    "    for opt in question.options:\n",
    "        cleaned_opt = opt.strip().lower()\n",
    "        if 'none of the above' in cleaned_opt or ('none' in cleaned_opt and 'above' in cleaned_opt) or cleaned_opt.startswith('none of') or cleaned_opt == 'neither' or 'none' in question.options[len(question.options)-1]:\n",
    "            return False\n",
    "    return True"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ff1b534-99e1-418d-920c-2f0ecc45a6bc",
   "metadata": {},
   "source": [
    "## All Of The Above\n",
    "Avoid all of the above options as students can guess correct responses based on partial information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "id": "c3084749-6f1b-42fb-9cd4-2b761b7d7b68",
   "metadata": {},
   "outputs": [],
   "source": [
    "def all_of_the_above(question):\n",
    "    for opt in question.options:\n",
    "        cleaned_opt = opt.strip().lower()\n",
    "        if 'all of the above' in cleaned_opt or ('all' in cleaned_opt and 'above' in cleaned_opt) or ('all if the' in cleaned_opt)  or ('all of the' in cleaned_opt):\n",
    "            return False\n",
    "\n",
    "    return True"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09faad15-79dc-4cfd-b276-28000edd7843",
   "metadata": {},
   "source": [
    "## Fill-In-The-Blank\n",
    "Avoid omitting words in the middle of the stem that students must insert from the options provided "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "id": "b00becc5-6f57-4466-b514-78a4527848ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Programming questions might contain a single underscore, so check for multiple\n",
    "def fill_in_the_blank(question):\n",
    "    if \"__\" in question.stem or ('fill in the blank' in question.stem.lower()):\n",
    "        return False\n",
    "    \n",
    "    return True"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bef9014d-8f97-45ef-bccc-ad6fd6f8c823",
   "metadata": {},
   "source": [
    "## True/False\n",
    "The options should not be a series of true/false statements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "id": "bfe524b1-9144-4b3d-a3db-67b572a20958",
   "metadata": {},
   "outputs": [],
   "source": [
    "def true_or_false(question):\n",
    "    options = question.options.copy()\n",
    "    \n",
    "    #Check for true & false mentioned in the stem\n",
    "    for sent in question.stem.split('.'):\n",
    "        sent = sent.lower()\n",
    "        if 'false' in sent and 'true' in sent:\n",
    "            return False    \n",
    "    \n",
    "    for opt in options:\n",
    "        cleaned_opt = opt.strip().lower() \n",
    "        if cleaned_opt == 'true' or cleaned_opt == 'false' or cleaned_opt == 'yes' or cleaned_opt == 'no':\n",
    "            return False\n",
    "\n",
    "    return True"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dcbc51e2-e837-41a3-b4c8-985ef27c671e",
   "metadata": {},
   "source": [
    "## Absolute Terms\n",
    "Avoid the use of absolute terms (e.g. never, always, all) in both the question stem as it can be confusing and the options as students are aware that they are almost always false"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "id": "11b7ca8e-36f7-4e6c-8ae1-7a244b70227d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#The list of absolute terms can be different for the stem and options, but we need to be careful here, as sometimes these can be used in proper ways\n",
    "absolutes = [\"always\", \"never\", \"none\", \"all\", \"completely\", \"absolutely\", \"totally\", \"definitely\", \"incapable\", \"inevitable\"]\n",
    "def absolute_terms(question):\n",
    "\n",
    "    #Check for terms in the question stem, if we we find any, have GPT-4 help us verify the use of it.\n",
    "    stem = question.stem.lower()\n",
    "    if any(word in stem.split() for word in absolutes):\n",
    "        if not true_or_false(question):\n",
    "            return True\n",
    "        else:\n",
    "            return absolute_terms_verify(stem)\n",
    "\n",
    "    #Check for terms in the options, if we we find any, have GPT-4 help us verify the use of it.\n",
    "    absolutes_options = [\"always\", \"never\", \"none\", \"completely\", \"absolutely\", \"totally\", \"definitely\", \"incapable\", \"inevitable\", \"all\"]\n",
    "    for opt in question.options:\n",
    "        cleaned_opt = opt.strip().lower()\n",
    "        \n",
    "        #Count all, which is a special case, but not in the case of \"all of the above\"\n",
    "        if any(word in cleaned_opt for word in absolutes_options):        \n",
    "            if none_of_the_above(question) and all_of_the_above(question) and true_or_false(question):\n",
    "                if \"all\" in cleaned_opt: \n",
    "                    return absolute_terms_verify(cleaned_opt)\n",
    "                return False\n",
    "    \n",
    "    return True\n",
    "\n",
    "def absolute_terms_verify(prompt):\n",
    "    sysrole = \"\"\"You are an expert educator evaluating multiple-choice questions for the presence of absolute terms. Your task is to examine the question’s text and answer options for any words or phrases that might give away the correct answer or help a student eliminate an incorrect answer due to their overly certain or extreme language.\"\"\"\n",
    "    prompt = \"\"\"\n",
    "Instructions:\n",
    "    Provide a single numeric score from 1 to 10, reflecting how the presence of absolute terms (e.g., “always”, “never”, “all”, “must”, \"none\", “only”) might help a student guess or eliminate incorrect options or if the question's stem contains absolute terms that are used in a way which constitutes a blanket generalization or hyperbole.\n",
    "\n",
    "    Scoring Guidelines:\n",
    "        1–3: The question or answer choices contain multiple or very strong absolute terms that greatly simplify finding the correct answer.\n",
    "        4–6: The question or choices contain some absolute terms that might help clue the student toward the answer.\n",
    "        7–9: The question or choices contain minimal absolute terms with limited impact on revealing the correct answer.\n",
    "        10: The question and all answer options are free of absolute terms that could reveal the answer.\n",
    "    \n",
    "    Your response should only be the number from 1 to 10. Do not include any explanation or additional text.\n",
    "\n",
    "Question: {}\n",
    "Options: {}\n",
    "Answer: {}\"\"\".format(question.stem, question.options, question.correct_option)\n",
    "\n",
    "    return llm_rating(sysrole, prompt)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "083615c2-0b27-43c3-8bf6-2b5bfe42444b",
   "metadata": {},
   "source": [
    "## Longest Answer Correct\n",
    "Often the correct option is longer and includes more detailed information, which clues students to this option"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "id": "5d4b8509-78f6-470a-8263-70f527edf130",
   "metadata": {},
   "outputs": [],
   "source": [
    "#If the correct answer is noticably longer (more than 25%) than the second longest answer, flag it.\n",
    "def longest_answer_correct(question):\n",
    "\n",
    "    #Ignore this criteria for True/False questions\n",
    "    if not true_or_false(question) or '[SEP]' in question.correct_option:\n",
    "        return True\n",
    "        \n",
    "    correct = question.correct_option\n",
    "    options = question.options.copy()\n",
    "    \n",
    "    # Split the correct answers (if multiple) and remove them from options\n",
    "    correct_answers = [opt.strip() for opt in question.correct_option.split('[SEP]')]\n",
    "    for opt in correct_answers:\n",
    "        if opt in options:\n",
    "            options.remove(opt)\n",
    "\n",
    "    longest_option = 0\n",
    "    for opt in options:\n",
    "        if len(opt) >= longest_option:\n",
    "            longest_option = len(opt)\n",
    "        \n",
    "    #If the longest option is only by 25% or it's a three words or less, then this passes\n",
    "    if longest_option >= len(correct) *.75 or len(correct.split()) < 4:\n",
    "        return True\n",
    "    \n",
    "    return False"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d3c2c16-62c1-41d9-891b-90d3f9e43b8d",
   "metadata": {},
   "source": [
    "## Negative worded\n",
    "Negatively worded stems are less likely to measure important learning outcomes and can confuse students"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "id": "95d72d49-6504-47d8-899f-cddcc4a1bab5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#The list of negative words can potentially cause this to be too restrictive, particularly for words such as can't and won't\n",
    "def negative_worded_stem(question):\n",
    "    negatives = [\"none\", \"never\", \"without\", \"exclude\", \"deny\", \"refuse\", \"oppose\", \"dispute\", \"can't\", \"won't\", \"not\"] \n",
    "\n",
    "    stem = question.stem.lower()\n",
    "    if any(word in stem.split() for word in negatives):\n",
    "        return False\n",
    "\n",
    "    for sent in question.stem.split('.'):\n",
    "        sent = sent.lower()        \n",
    "        if 'which' in sent and ('false' in sent or 'not' in sent or 'incorrect' in sent or 'except' in sent) or \\\n",
    "        'what' in sent and ('false' in sent or 'not' in sent or 'incorrect' in sent or 'except' in sent):\n",
    "            return False    \n",
    "    \n",
    "    return True"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bfbf7aa1-71aa-4895-beca-ad0aec7b7a05",
   "metadata": {},
   "source": [
    "## Word Repeats\n",
    "Avoid similarly worded stems and correct responses or words repeated in the stem and correct response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 347,
   "id": "fb0bbeb0-01b2-4045-9b5d-762d6c39fb81",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Find the nouns in question.correct_option and question.stem --> stem them --> compare cosine similiary (using sentence transformer)\n",
    "#Also check for the synonyms, compare them. However, if the word(s) are used in the other options, then it's fine.\n",
    "#Nouns: NN noun, singular ‘- desk’, NNS noun plural – ‘desks’, NNP proper noun, singular – ‘Harrison’, NNPS proper noun, plural – ‘Americans’ \n",
    "lemmatizer = WordNetLemmatizer()\n",
    "nouns = ['NN', 'NNS', 'NNP', 'NNPS']\n",
    "\n",
    "def word_repeats_in_stem_and_correct_answer(question):   \n",
    "    options = question.options.copy()\n",
    "    \n",
    "    all_options = ' '.join(options)\n",
    "    \n",
    "    # Split the correct answers (if multiple) and remove them from options\n",
    "    correct_answers = [opt.strip() for opt in question.correct_option.split('[SEP]')]\n",
    "    for opt in correct_answers:\n",
    "        if opt in options:\n",
    "            options.remove(opt)\n",
    "        \n",
    "    #This code checks for matching words, specifically nouns and verbs, between the correct answer and stem\n",
    "    word_types = ['NN', 'NNS', 'NNP', 'NNPS', 'VB', 'VBG', 'VBN', 'VBP', 'VBZ']\n",
    "    stem = strip_punctuation(question.stem)\n",
    "    matching_words = []\n",
    "    for wrd in stem.split():\n",
    "        if wrd not in stop_words and wrd in question.correct_option:\n",
    "            matching_words.append(wrd)\n",
    "            \n",
    "    matching_words = list(set(matching_words)).copy()\n",
    "    matching_words_copy = matching_words.copy()\n",
    "    for wrd in matching_words:\n",
    "        for opt in options:\n",
    "            if wrd in opt:\n",
    "                matching_words_copy.remove(wrd)\n",
    "                break\n",
    "\n",
    "    #If the word is longer than 4 characters, because non-matching verbs/nouns of smaller characters typically are not cues\n",
    "    if len([s for s in matching_words_copy if len(s) >= 4]) > 0:\n",
    "        again = []\n",
    "        tagged = nltk.pos_tag(matching_words_copy)\n",
    "        for t in tagged:\n",
    "            if t[1] in word_types:\n",
    "                again.append(t[0].lower())\n",
    "        if len(again) > 0:\n",
    "            if '[SEP]' in question.correct_option:\n",
    "                for mwc in matching_words_copy:\n",
    "                    if mwc in question.correct_option.split('[SEP]')[0] and mwc in question.correct_option.split('[SEP]')[1]:\n",
    "                        print('*** SEP')\n",
    "                        return False\n",
    "                    else:\n",
    "                        return True\n",
    "            else:\n",
    "                #There's the potential false positive where all answer choices are repeated in the question's stem\n",
    "                all_ops_in_stem = 0\n",
    "                for opt in question.options:\n",
    "                    opt = opt.lower()\n",
    "                    stem = question.stem.lower()\n",
    "                    if opt in stem:\n",
    "                        all_ops_in_stem = all_ops_in_stem + 1\n",
    "                if all_ops_in_stem == len(question.options):\n",
    "                    return True\n",
    "                \n",
    "                return False\n",
    "    return True\n",
    "\n",
    "def strip_punctuation(text):\n",
    "    return ''.join(char for char in text if char not in string.punctuation)\n",
    "\n",
    "#This is now used for Logical Cue\n",
    "def get_lemma_nouns(text):\n",
    "    all_nouns = []\n",
    "    tokenized = sent_tokenize(text)\n",
    "    \n",
    "    for i in tokenized:\n",
    "\n",
    "        # Word tokenizers is used to find the words and punctuation in a string\n",
    "        wordsList = nltk.word_tokenize(i)\n",
    "\n",
    "        # removing stop words from wordList\n",
    "        wordsList = [w for w in wordsList if not w in stop_words]\n",
    "\n",
    "        # Using a Tagger. Which is part-of-speech tagger or POS-tagger.\n",
    "        tagged = nltk.pos_tag(wordsList)\n",
    "        \n",
    "        # Add any nouns to this list\n",
    "        for t in tagged:\n",
    "            if t[1] in nouns:\n",
    "                all_nouns.append(t[0].lower())\n",
    "    \n",
    "    lemmatized_nouns = []\n",
    "    for n in all_nouns:\n",
    "        lemmatized_word = lemmatizer.lemmatize(n, pos=\"n\")\n",
    "        lemmatized_nouns.append(lemmatized_word.lower())\n",
    "    \n",
    "    return lemmatized_nouns   \n",
    "\n",
    "def word_repeats_verify(question):\n",
    "    options = question.options.copy()\n",
    "    \n",
    "    # Split the correct answers (if multiple) and remove them from options\n",
    "    correct_answers = [opt.strip() for opt in question.correct_option.split('[SEP]')]\n",
    "    for opt in correct_answers:\n",
    "        if opt in options:\n",
    "            options.remove(opt)\n",
    "        \n",
    "    sysrole = \"\"\"You are an expert educator evaluating multiple-choice questions to identify whether a unique key word, phrase, or term is shared exclusively between the stem and the correct answer. Such repetition can provide an unintended clue for testwise students. If no exclusive repetition of a unique key word or phrase exists, the question should be considered free of this issue.\"\"\"\n",
    "    prompt = \"\"\"\n",
    "Instructions:\n",
    "    Respond with a score from 1–10 based on how prominently a unique key word, phrase, or term from the stem is repeated only in the correct answer (not appearing in any other options). Use the following Scoring Guidelines:\n",
    "\n",
    "    Scoring Guidelines:\n",
    "        1–3: A clearly noticeable or direct match of a unique key term between the stem and the correct answer makes the correct option stand out strongly.\n",
    "        4–6: Some repetition of a key term provides a potential clue but is not blatantly revealing.\n",
    "        7–9: Only minimal or subtle overlap in wording, unlikely to give a significant advantage to a testwise student.\n",
    "        10: No unique repeated key words, phrases, or terms appear exclusively in the stem and the correct answer.\n",
    "\n",
    "    Do not lower the score for common or generic terms (e.g., articles, conjunctions, basic verbs) unless they serve as an unmistakable clue.\n",
    "    \n",
    "    Only penalize repeated, uniquely identifying words or phrases that are exclusive to the stem and correct answer.\n",
    "    \n",
    "    If no exclusive repetition is found, assign a score of 10.\n",
    "\n",
    "    Your response should only be the number from 1 to 10. Do not include any explanation or additional text.\n",
    "\n",
    "Question Stem: {}\n",
    "Options: {}\n",
    "Answer: {}\"\"\".format(question.stem, options, question.correct_option)\n",
    "    \n",
    "    return llm_rating(sysrole, prompt)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a49c0c9-68fd-4ed5-b307-cd55228c6a1e",
   "metadata": {},
   "source": [
    "## Logical Cue - This one is challenging, requires domain knowledge."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 348,
   "id": "6a77330d-515c-4219-8691-edd5c6db81b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#An example of a logical cue is asking students to select the most appropriate pharmaceutical intervention for a problem and only having one or two options which\n",
    "#Using NER, if the question asks for a <certain type of noun, like a <person> then the options should all be <people> too.\n",
    "\n",
    "def avoid_logical_cues(question):\n",
    "    options = question.options.copy()\n",
    "\n",
    "    # Split the correct answers (if multiple) and remove them from options\n",
    "    correct_answers = [opt.strip() for opt in question.correct_option.split('[SEP]')]\n",
    "    for opt in correct_answers:\n",
    "        if opt in options:\n",
    "            options.remove(opt)\n",
    "    \n",
    "    if len(options) < 2:\n",
    "        return True\n",
    "   \n",
    "    lemma_nouns_answ = get_lemma_nouns(question.correct_option)\n",
    "    lemma_nouns_options = [get_lemma_nouns(opt) for opt in options]\n",
    "        \n",
    "    entities_in_options = []\n",
    "    for opt in lemma_nouns_options:\n",
    "        for val in opt:\n",
    "            doc = nlp(val)\n",
    "            if doc.ents:\n",
    "                entities_in_options.append(doc.ents[0].label_)\n",
    "    \n",
    "    entities_in_answer = []\n",
    "  \n",
    "    for val in lemma_nouns_answ:\n",
    "        doc = nlp(val)\n",
    "        \n",
    "        #If the noun(s) in the answer choice can be tagged with an entity\n",
    "        if doc.ents:\n",
    "            answer_entity = doc.ents[0].label_\n",
    "            if answer_entity not in entities_in_options:\n",
    "                return logical_cue_verify(question)\n",
    "\n",
    "    \n",
    "    #If the stem has a number and only one option has a number\n",
    "    numbers_in_stem = extract_all_numerical_values(question.stem)\n",
    "    numbers_in_options = sum(1 for opt in options if extract_all_numerical_values(opt))\n",
    "\n",
    "    if len(numbers_in_stem) > 0 and numbers_in_options == 1:\n",
    "        return False\n",
    "    \n",
    "    return True\n",
    "\n",
    "def logical_cue_verify(question):\n",
    "    sysrole = \"\"\"You are an expert educator evaluating multiple-choice questions for any unintended hints or logical cues in the stem and answer choices that might allow a testwise student to identify the correct option without true mastery of the content. Your primary objective is to spot clues, patterns, or giveaways that compromise the fairness or validity of the question.\"\"\"\n",
    "    prompt = \"\"\"\n",
    "Instructions:\n",
    "    Respond with a score from 1–10 based on how likely it is that a testwise student could identify the correct answer through logical cues, based on the following Scoring Guidelines:\n",
    "\n",
    "    Scoring Guidelines:\n",
    "        1–3: The question or options contain significant or obvious clues, making it easy for a savvy testwise student to guess the correct answer.\n",
    "        4–6: The question or options have some identifiable cues, but they are less obvious and require closer attention to exploit.\n",
    "        7–9: The question is generally well-constructed, with only minor or subtle cues that might help a testwise student.\n",
    "        10: The question is free of any noticeable cues, providing no advantages to a student guessing logically.\n",
    "\n",
    "    Please do not lower the score for common question-design elements that are unavoidable or do not genuinely compromise fairness.  \n",
    "\n",
    "    Your response should only be a single number from 1 to 10, with no additional explanation.\n",
    "\n",
    "Question: {}\n",
    "Options: {}\n",
    "Answer: {}\"\"\".format(question.stem, question.options, question.correct_option)\n",
    "\n",
    "    return llm_rating(sysrole, prompt)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7005fd19-41f4-44a6-9f70-6720442b7299",
   "metadata": {},
   "source": [
    "## Lost Sequence\n",
    "If options are numerical, they should go lowest to highest or vice-versa, not a random arrnagement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "id": "80c25d69-3a7b-476b-a6a1-12f7c0d09bb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#If answer choices are numeric, sort them, compare to current order\n",
    "#If all but one are numerical, make sure they are in order and the \"word\" option is last.\n",
    "\n",
    "def lost_sequence(question):\n",
    "    options = question.options.copy() \n",
    "    opts = []\n",
    "    non_numerical_option = 0\n",
    "    for opt in options:\n",
    "        #First check for fractions\n",
    "        fraction = extract_fraction_to_float(opt)\n",
    "        if fraction:\n",
    "            opts.append(fraction)\n",
    "        else:\n",
    "            val = extract_all_numerical_values(opt)\n",
    "            if len(val) == 1:\n",
    "                opts.append(float(val[0].replace(',', '')))\n",
    "            else:\n",
    "                non_numerical_option = non_numerical_option + 1\n",
    "    \n",
    "    if non_numerical_option > 0 and not(non_numerical_option == 1 and len(opts) == len(options)-1):\n",
    "        return True\n",
    "\n",
    "    float_options = [float(x) for x in opts]    \n",
    "    sorted_options = sorted(float_options)\n",
    "    reverse_sorted_options = sorted(float_options, reverse=True)\n",
    "    \n",
    "    if sorted_options == float_options:\n",
    "        #Numeric options are sorted\n",
    "        return True\n",
    "    elif reverse_sorted_options == float_options:\n",
    "        #Numeric options are sorted in reverse order, which might make sense for the question\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "\n",
    "def extract_all_numerical_values(s):\n",
    "    pattern = r'-?\\d*(?:,\\d{3})*\\.\\d+|-?\\d+(?:,\\d{3})*'\n",
    "    return re.findall(pattern, s)\n",
    "\n",
    "# Regex pattern to match fractions with optional decimal numerator and/or denominator\n",
    "def extract_fraction_to_float(s):\n",
    "    pattern = r'-?\\b\\d+(\\.\\d+)?/\\d+(\\.\\d+)?\\b'\n",
    "    match = re.search(pattern, s)\n",
    "    if match:\n",
    "        a , b = match.group().split(\"/\")\n",
    "        fraction = float(a) / float(b)\n",
    "        return fraction\n",
    "    else:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7cfd3dae-ff12-44cb-b30c-1df276d79aa0",
   "metadata": {},
   "source": [
    "## More Than One Correct\n",
    "This updated approach simplifies the task by directing GPT-4o to focus solely on identifying the correct answers. This reduction in complexity likely minimizes cognitive strain on the model, allowing it to perform more effectively and with greater accuracy. By removing the secondary task of evaluating the number of correct answers, the model can concentrate fully on its core strength: understanding and answering the question based on its underlying knowledge and reasoning. - Gilles Chen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "id": "c77fe16e-5cc5-4522-9b08-86995409fdee",
   "metadata": {},
   "outputs": [],
   "source": [
    "format_mtoc ={\n",
    "    \"Stem\": \"The Question\",\n",
    "    \"correct_answers\": [\n",
    "        {\n",
    "            \"A\": \"The choice\"\n",
    "        },\n",
    "        {\n",
    "            \"C\": \"The choice\"\n",
    "        }\n",
    "    ]\n",
    "}\n",
    "\n",
    "def more_than_one_correct(question):\n",
    "    # Ensure the question has at least 4 options\n",
    "    if len(question.options) < 2:\n",
    "        raise ValueError(\"A multiple-choice question must have at least two options.\")\n",
    "    \n",
    "    sysrole = \"\"\"You are an expert and an astute instructor. \n",
    "    Given a multiple-choice question and possible answers, determine the correct answers. \n",
    "    Your reply must be in the following JSON format: {}\"\"\".format(format_mtoc)\n",
    "\n",
    "    # Dynamically construct the answer choices\n",
    "    answer_choices = \"\\n\".join([f\"{chr(65+i)}: {option}\" for i, option in enumerate(question.options)])\n",
    "\n",
    "    # Create the prompt\n",
    "    prompt = f\"\"\"\n",
    "        question: {question.stem}\n",
    "        \n",
    "        answers:\n",
    "        {answer_choices}\n",
    "    \"\"\"\n",
    "\n",
    "    done = False\n",
    "    expert_reasoning = 'blank'\n",
    "\n",
    "    while not done:\n",
    "        try:\n",
    "            response = client.chat.completions.create(\n",
    "              response_format={\"type\": \"json_object\"},\n",
    "              model=model_engine,\n",
    "              messages=[\n",
    "                 {\"role\": \"system\", \"content\": sysrole},\n",
    "                {\"role\": \"user\", \"content\": prompt},\n",
    "              ],\n",
    "              max_tokens = 4096,\n",
    "              temperature = 0.7\n",
    "             )\n",
    "            done = True \n",
    "        except Exception as error:\n",
    "            print('errored in LLM API call: ', error)\n",
    "            time.sleep(10)\n",
    "\n",
    "    expert_reasoning = json.loads(response.choices[0].message.content)\n",
    "\n",
    "    return len(expert_reasoning[\"correct_answers\"]) == 1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c89088a-9ee3-4c3a-bc05-bb0fb346d212",
   "metadata": {},
   "source": [
    "## Complex or K-type\n",
    "Avoid questions that have a range of correct responses, that ask students to select from a number of possible combinations of the responses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "id": "f53332fb-6ad0-431d-99fb-1f39e25914da",
   "metadata": {},
   "outputs": [],
   "source": [
    "#If the answer options share the same words between one another and there are commas present then it's k type\n",
    "def complex_k_type(question):    \n",
    "    options = question.options.copy()\n",
    "    \n",
    "    correct_answers = [opt.strip() for opt in question.correct_option.split('[SEP]')]\n",
    "    \n",
    "    # Ensure all correct answers are removed safely\n",
    "    for opt in correct_answers:\n",
    "        if opt in options:\n",
    "            options.remove(opt)\n",
    "\n",
    "    # If the question is not an \"all of the above\", \"none of the above\", or \"true/false\" type, return True\n",
    "    if not (all_of_the_above(question) or none_of_the_above(question) or true_or_false(question)):\n",
    "        return True \n",
    "\n",
    "    # If fewer than 2 options remain after removing correct answers, return True\n",
    "    if len(options) < 2:\n",
    "        return True \n",
    "\n",
    "    # Check if all remaining options contain a comma\n",
    "    all_contain_comma = all(',' in opt for opt in options)\n",
    "\n",
    "    # Extract lemma nouns from the correct answer and options\n",
    "    lemma_nouns_answ = get_lemma_nouns(question.correct_option)\n",
    "    lemma_nouns_options = [get_lemma_nouns(opt) for opt in options]\n",
    "\n",
    "    # Count options that share at least one noun with the correct answer\n",
    "    options_that_share_noun = sum(\n",
    "        bool(set(lno) & set(lemma_nouns_answ)) for lno in lemma_nouns_options\n",
    "    )\n",
    "\n",
    "    # Check if all options are variations of \"Yes\" or \"No\" answers\n",
    "    yes_no_options = sum(opt.lower().startswith((\"yes\", \"no\")) for opt in options)\n",
    "    if yes_no_options == len(options):\n",
    "        return True\n",
    "\n",
    "    # If options share a key noun, contain multiple nouns, and include commas, it might be a K-type question\n",
    "    if options_that_share_noun > 0 and all_contain_comma:\n",
    "        return False\n",
    "\n",
    "    # Normalize options by removing list-like formatting\n",
    "    cleaned_options = [clean_string(opt) for opt in options]\n",
    "\n",
    "    # Check if all cleaned options contain the same words\n",
    "    options_word_sets = [set(opt.split()) for opt in cleaned_options]\n",
    "    \n",
    "    if len(set(map(frozenset, options_word_sets))) == 1:\n",
    "        return False\n",
    "\n",
    "    return complex_k_type_verify(question)\n",
    "\n",
    "def clean_string(string):\n",
    "    # remove whitespace\n",
    "    cleaned_string = string.strip()\n",
    "    \n",
    "    # remove punctuation\n",
    "    cleaned_string = re.sub(r'[^\\w\\s]', '', cleaned_string)\n",
    "    \n",
    "    # remove list notation\n",
    "    cleaned_string = re.sub(r'\\b(i|ii|iii|iv|v|vi|vii|viii|ix|x|xi|xii|xiii|xiv|xv|xvi|xvii|xviii|xix|xx)\\b', '', cleaned_string)\n",
    "    cleaned_string = re.sub(r'\\b(A|B|C|D|E|F)\\b', '', cleaned_string)\n",
    "    return cleaned_string\n",
    "\n",
    "\n",
    "def complex_k_type_verify(question):\n",
    "    sysrole = \"\"\"You are an expert educator and assessment designer specializing in evaluating complex multiple-choice questions. Your primary task is to identify K-Type (Complex) multiple-choice questions. These questions require students to select from a range of combinations of responses, often using phrases like \"and,\" \"or,\" \"only,\" or presenting multiple items and pairs separated by commas, semicolons, periods, or numbers.\n",
    "    \n",
    "    K-Type questions are characterized by options that represent different combinations of correct or partially correct answers. Unlike standard multiple-choice questions that have a single correct answer, K-Type questions involve evaluating multiple statements to determine which combination is valid.\n",
    "    \n",
    "    Your goal is to identify questions that fit this K-Type structure and distinguish them from traditional single-answer multiple-choice questions.\"\"\"\n",
    "\n",
    "    prompt = \"\"\"\n",
    "Instructions:\n",
    "    Evaluate the multiple-choice question below to determine if it qualifies as a K-Type (Complex) question, where students are required to choose from a two or more combinations of responses, rather than a single correct answer, often using phrases like \"and,\" \"or,\" \"only,\" or presenting multiple items and pairs separated by commas, semicolons, periods, or numbers.\n",
    "    \n",
    "    A K-Type question should have options that share different combinations of the same answer, just because an option has a few repeated keywords or several commas, does not mean it is K-Type.\n",
    "    \n",
    "    Provide a score from 1 to 10 based on the following Scoring Guidelines:\n",
    "    \n",
    "    Scoring Guidelines:\n",
    "        1-3: The question is clearly a K-Type question, where options represent different combinations of possible responses. These options often use phrases like \"and\", \"or\", \"only\", or lists and pairings separated by commas or semicolons. The question requires students to assess multiple statements and determine which combination is valid.\n",
    "        4-6: The question shows strong characteristics of a K-Type question, but the combinations may be less explicit or not as complex.\n",
    "        7-9: The question has some elements of a K-Type structure, but it is closer to a traditional multiple-choice question with a single correct answer.\n",
    "        10: The question is not a K-Type question. It is a standard multiple-choice question that requires selecting a single correct answer without evaluating combinations of responses.\n",
    "    \n",
    "    Your response should be a single number between 1 and 10, without any additional explanation or rationale.\n",
    "\n",
    "Question: {}\n",
    "Options: {}\n",
    "Answer: {}\"\"\".format(question.stem, question.options, question.correct_option)\n",
    "\n",
    "    return llm_rating(sysrole, prompt)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "11a46d89-56a8-490a-a09c-ebd1cb710e2a",
   "metadata": {},
   "source": [
    "## Ambiguous or Unclear Information\n",
    "Questions and all options should be written in clear, unambiguous language"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "id": "951f058f-3edb-4619-9758-f6a26ae87a61",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at Ashishkr/query_wellformedness_score were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']\n",
      "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "#Roberta model from: https://huggingface.co/cointegrated/roberta-large-cola-krishna2020\n",
    "#In addition to cola, added a query wellformedness score metric from https://huggingface.co/Ashishkr/query_wellformedness_score \n",
    "cola = pipeline('text-classification', model='cointegrated/roberta-large-cola-krishna2020',truncation=True)\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"Ashishkr/query_wellformedness_score\", ignore_mismatched_parameters=True)\n",
    "model = AutoModelForSequenceClassification.from_pretrained(\"Ashishkr/query_wellformedness_score\")\n",
    "def check_string(s):\n",
    "    # Check for presence of '<' or '>'\n",
    "    if '<' in s or '>' in s:\n",
    "        return True\n",
    "\n",
    "    # Check for two or more ':'\n",
    "    if len(re.findall(r':', s)) >= 2:\n",
    "        return True\n",
    "\n",
    "    # Check for two or more ':'\n",
    "    if len(re.findall(r'=', s)) >= 2:\n",
    "        return True\n",
    "        \n",
    "    return False\n",
    "\n",
    "scores_list = []\n",
    "\n",
    "def ambiguous_unclear_information(question):\n",
    "\n",
    "    sysrole = \"\"\"You are an expert educator evaluating the clarity of multiple-choice questions (MCQs) for ambiguity or comprehension issues. Your primary goal is to assess whether a well-prepared student would easily grasp the intended meaning of the question and confidently select the correct answer.\"\"\"\n",
    "\n",
    "    prompt= \"\"\"\n",
    "Instructions:\n",
    "    Respond with a score of 1 - 10, where a 1 indicates a high presence of ambiguous or unclear information and 10 indicates an absence of ambiguity and a clear question.\n",
    "\n",
    "    Scoring Guidelines:\n",
    "        1–3: The question is severely ambiguous, unclear, or confusing, even for a knowledgeable student.\n",
    "        4–6: The question has noticeable clarity issues or structural problems that may impede understanding, even when looking at the options.\n",
    "        7–9: The question is mostly clear and understandable, with only minor or occasional issues that do not significantly hinder comprehension.\n",
    "        10:  The question is fully clear and understandable, despite any minor imperfections or complexities.\n",
    "\n",
    "    Focus on major clarity issues that could seriously hinder comprehension.\n",
    "       - Do not penalize for minor typos, grammatical issues, or stylistic preferences unless they cause confusion.\n",
    "       - Standard phrasing like \"Which of the following...\" is acceptable if the options provide enough context.\n",
    "       - Technical terms or challenging vocabulary are acceptable if they would be understandable to a knowledgeable student.\n",
    "\n",
    "    If the question depends on the answer options for clarity, this is acceptable as long as the overall meaning is still discernible given the options. In other words, the combination of the stem and options should clearly convey what the question is asking.\n",
    "\n",
    "    Your response should only be a value in the range of 1 - 10, do not include an explanation or rationale.\n",
    "\n",
    "Question: {}\n",
    "Answer: {}\n",
    "Options: {}\"\"\".format(question.stem, question.correct_option, question.options)\n",
    "    \n",
    "    return llm_rating(sysrole, prompt)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2738e90-af92-4bb8-ae2c-54cf85d51394",
   "metadata": {},
   "source": [
    "## Gratuitous Information\n",
    "Avoid unnecessary information in the stem that is not required to answer the question"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "id": "7f11f16f-8c9d-487c-9580-f8fa4460ca07",
   "metadata": {},
   "outputs": [],
   "source": [
    "grat_scores_list = []\n",
    "\n",
    "def gratuitous_information_in_stem(question):  \n",
    "    #How effective are lexical richness measures for differentiations of vocabulary proficiency? A comprehensive examination with clustering analysis\n",
    "    #From: https://github.com/LSYS/LexicalRichness\n",
    "    stem = LexicalRichness(question.stem)\n",
    "    \n",
    "    if stem.cttr > 4.5:\n",
    "        scores_list.append(1)\n",
    "        return False\n",
    "    \n",
    "    return gratuitous_information_in_stem_verify(question)\n",
    "\n",
    "\n",
    "def gratuitous_information_in_stem_verify(question):\n",
    "    sysrole = \"\"\"You are an expert educator evaluating multiple-choice questions for the presence of gratuitous or unnecessary information in the question's text that could confuse students. Your focus is on identifying significant issues where extraneous details might distract or mislead a student, not on minor additional information that doesn't impact understanding.\"\"\"\n",
    "    prompt = \"\"\"\n",
    "Instructions:\n",
    "    Respond with a score of 1 - 10, where 1 means the question contains significant gratuitous information that could confuse students, and 10 means the question is concise and contains only relevant information necessary for understanding and answering the question correctly.\n",
    "\n",
    "    Scoring Guidelines:\n",
    "        1–3: The question contains excessive or irrelevant details that significantly distract or confuse students.\n",
    "        4–6: The question contains noticeable extraneous information that may hamper clarity but does not severely mislead.\n",
    "        7–9: The question is mostly concise, with minor additional details that do not distract from the main point.\n",
    "        10: The question is entirely focused on essential information needed to answer correctly, with no unnecessary details.\n",
    "\n",
    "    Do not assign a low score (1–3 or 4–6) for minor details that are typical in multiple-choice questions and do not affect comprehension.  \n",
    "    \n",
    "    Do not assign a low score if the question is scenario-based and the details meaningfully contribute to the scenario.\n",
    "\n",
    "    Your response should be **only** a single integer from 1–10. Do not include any explanation or rationale.\n",
    "\n",
    "Question: {}\n",
    "Options: {}\n",
    "Answer: {}\"\"\".format(question.stem, question.options, question.correct_option)\n",
    "\n",
    "    return llm_rating(sysrole, prompt)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4b4ff8d-6d6a-45bd-a52c-62ede6ee349b",
   "metadata": {},
   "source": [
    "## Convergence Cues\n",
    "Avoid convergence cues in options where there are different combinations of multiple components to the answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "id": "21a2d7e4-f1fe-4328-bc2d-e3e40626fa0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Check for synonyms, because they'll know it's the word they've most recently come across in the text\n",
    "#The correct option is likely to be used more (when in pairs, etc.) --> k-type (super similar by description)\n",
    "def avoid_convergence_cues(question):\n",
    "    \n",
    "    # Copy options to avoid modifying the original question object\n",
    "    options = question.options.copy()\n",
    "    \n",
    "    # Split the correct answers (if multiple) and remove them from options\n",
    "    correct_answers = [opt.strip() for opt in question.correct_option.split('[SEP]')]\n",
    "    for opt in correct_answers:\n",
    "        if opt in options:\n",
    "            options.remove(opt)\n",
    "    \n",
    "    # Ensure there are enough distractors for meaningful analysis\n",
    "    if len(options) < 2:  \n",
    "        return True  # Not enough distractors to compare\n",
    "    \n",
    "    # Get lemmas (base forms) of nouns in the correct answer(s)\n",
    "    lemma_nouns_answ = set()\n",
    "    for answer in correct_answers:\n",
    "        lemma_nouns_answ.update(get_lemma_nouns(answer))\n",
    "    \n",
    "    # Get lemmas for all distractors\n",
    "    lemma_nouns_options = [set(get_lemma_nouns(opt)) for opt in options]\n",
    "    \n",
    "    # Collect synonyms of the correct answer(s)\n",
    "    synonyms = set()\n",
    "    for noun in lemma_nouns_answ:\n",
    "        for syn in wn.synsets(noun):\n",
    "            synonyms.update(l.name().lower().replace('_', ' ') for l in syn.lemmas())\n",
    "\n",
    "    # Check if any synonym appears in only some of the distractors\n",
    "    for opt in lemma_nouns_options:\n",
    "        repeating_nouns_synonyms = synonyms.intersection(opt)\n",
    "        if repeating_nouns_synonyms:\n",
    "            for rns in repeating_nouns_synonyms:\n",
    "                # Ensure the term is not present in every distractor (i.e., it's unevenly distributed)\n",
    "                if not all(rns in value for value in lemma_nouns_options):\n",
    "                    return avoid_convergence_cues_verify(question)\n",
    "                    \n",
    "    return True\n",
    "\n",
    "def avoid_convergence_cues_verify(question):\n",
    "    sysrole = \"\"\"You are an expert in educational assessment specializing in detecting convergence cues in multiple-choice questions. Convergence cues are present when the options share overlapping elements, refer to each other, or include combinations of other options (e.g., \"A and B,\" \"B or C\"). Your goal is to identify problematic convergence cues that are based on logical overlap between options (e.g., subsets, combinations, or dependencies). However, you should not flag distinct, independent numerical values or factual answers (like scientific measurements) as convergence cues unless they are structured to confuse students by implying a relationship between the options.\"\"\"\n",
    "    prompt = \"\"\"\n",
    "Instructions:\n",
    "    Respond with a score of 1 - 10, where a 1 indicates a high presence of convergence cues and 10 indicates an absence of convergence cues.\n",
    "\n",
    "    Scoring Guidelines:  \n",
    "        1-3: The options are heavily interrelated, sharing combinations or subsets that could confuse students (e.g., “A and B”, “B and C”, “A or C”).  \n",
    "        4-6: There is moderate convergence, with some overlap in the options but not enough to significantly confuse students.  \n",
    "        7-9: Minimal convergence cues; the options are distinct and clearly differentiated.  \n",
    "        10: No convergence cues detected; each option is unique and stands on its own without reference to other options.\n",
    "\n",
    "    Your response should only be a value in the range of 1 - 10, do not include an explanation or rationale.\n",
    "    \n",
    "Question: {}\n",
    "Options: {}\n",
    "Answer: {}\"\"\".format(question.stem, question.options, question.correct_option)\n",
    "\n",
    "    return llm_rating(sysrole, prompt)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71bc7cec-899d-4b12-8696-6db9e5d9a94f",
   "metadata": {},
   "source": [
    "## Grammatical Cues\n",
    "All options should be grammatically consistent with the stem and should be parallel in style and form"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "id": "8fcce92b-cf06-4601-a464-74f840592c29",
   "metadata": {},
   "outputs": [],
   "source": [
    "#If verb exists in answer choice, ensure it's the same tense as verb in other options\n",
    "#We want the stem to be the same, but as long as all the answers are the same, then it's fine, to avoid false positive.\n",
    "#https://huggingface.co/Unbabel/gec-t5_small\n",
    "def grammatical_cues_in_stem(question):\n",
    "    answer_tense = get_verb_tense(question.correct_option)\n",
    "\n",
    "    options = question.options.copy()\n",
    "\n",
    "    # Split the correct answers (if multiple) and remove them from options\n",
    "    correct_answers = [opt.strip() for opt in question.correct_option.split('[SEP]')]\n",
    "    for opt in correct_answers:\n",
    "        if opt in options:\n",
    "            options.remove(opt)\n",
    "    \n",
    "    for opt in options:\n",
    "        opt_tense = get_verb_tense(opt)    \n",
    "        if opt_tense != 'none' and answer_tense != 'none' and answer_tense is not opt_tense:\n",
    "            return False\n",
    "            \n",
    "    return grammatical_cues_verify(question)\n",
    "\n",
    "#Longer options might contain verbs of different tenses.\n",
    "#We want options that specifically have a single tense (past or present) and for it to be consistent with all other options.\n",
    "def get_verb_tense(text):\n",
    "    verbs = []\n",
    "    doc = nlp(text)\n",
    "    for token in doc:\n",
    "        if token.pos_ == 'VERB':\n",
    "            if token.tag_ in ['VBP', 'VBZ']:\n",
    "                verbs.append('present')\n",
    "            elif token.tag_ in ['VBD', 'VBN']:\n",
    "                verbs.append('past')\n",
    "            else:\n",
    "                verbs.append('none')\n",
    "\n",
    "    verb_tenses = list(set(verbs))\n",
    "    if len(verb_tenses) == 1 and verb_tenses[0] == 'past':\n",
    "        return 'past'\n",
    "    elif len(verb_tenses) == 1 and verb_tenses[0] == 'present':\n",
    "        return 'present'\n",
    "    return 'none'\n",
    "\n",
    "def grammatical_cues_verify(question):\n",
    "    sysrole = \"\"\"You are an expert educator evaluating multiple-choice questions for any grammatical cues in the stem and answer choices that could inadvertently reveal the correct answer or mislead students. This includes mismatched tenses, pronoun inconsistencies, lack of parallelism in structure, or other errors that a testwise student might exploit to identify or eliminate certain options without relying on content knowledge.\"\"\"\n",
    "    prompt = \"\"\"\n",
    "Instructions:\n",
    "    Respond with a score from 1–10 based on the overall grammatical consistency and parallelism of the question and its options. Use the following Scoring Guidelines:\n",
    "\n",
    "    Scoring Guidelines:\n",
    "        1–3: The question and/or options contain noticeable grammatical or structural inconsistencies that strongly hint at the correct answer.\n",
    "        4–6: Some grammatical mismatches or parallelism issues exist, but these are not glaring enough to immediately reveal the correct option.\n",
    "        7–9: The question and options are mostly consistent, with only minor discrepancies that are unlikely to provide a significant clue.\n",
    "        10: The question and options are grammatically consistent, parallel in style, and contain no identifiable cues that would give an advantage to a testwise student.\n",
    "\n",
    "    Do not lower the score for trivial wording differences that are typical and do not compromise fairness.\n",
    "    \n",
    "    Only penalize clear grammatical or structural errors that could be used to identify the correct option.\n",
    "\n",
    "    Your response should be only a single integer (1–10). Do not provide any explanations.\n",
    "\n",
    "Question: {}\n",
    "Options: {}\n",
    "Answer: {}\"\"\".format(question.stem, question.options, question.correct_option)\n",
    "    \n",
    "    #It's so strict on this, we're setting the threshold to 4 instead of 5.\n",
    "    #We get better results when we just reutnr True instead of verify, but the LLM could be sided with in these cases.\n",
    "    return llm_rating(sysrole, prompt,4)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "116522a0-b405-47e1-99ce-300161bd18ad",
   "metadata": {},
   "source": [
    "## Vague Terms\n",
    "Avoid the use of vague terms (e.g. frequently, occasionally) in the options as there is seldom agreement on their actual meaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "id": "8fd2f608-b5db-4b93-9617-2f15fbb1d531",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Like the ohter criteria that use a list of terms, these can be modified\n",
    "def vague_terms(question):\n",
    "    vagues = [\"frequently\", \"occasionally\", \"rarely\", \"seldom\", \"sometimes\", \"usually\", \"regularly\", \"periodically\", \"infrequently\", \"generally\", \"nearly\", \"more or less\", \"somewhat\", \"partly\"]\n",
    "    \n",
    "    #check the options then check the stem\n",
    "    for opt in question.options:\n",
    "        opt = opt.lower()\n",
    "        if any(word in opt for word in vagues):\n",
    "            return False\n",
    "\n",
    "    #In particular, these words can sometimes be used in the stem in a way that is not a flaw, but more likely than not, it is\n",
    "    if any(word in question.stem.lower() for word in vagues):\n",
    "        return False\n",
    "    \n",
    "    return True"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80a44d86-528b-44c3-8d36-9220092ea637",
   "metadata": {},
   "source": [
    "## Unfocused Stem\n",
    "The stem should present a clear and focused question that can be understood and answered without looking at the options"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "id": "14287260-5c60-4038-90ca-5b10caa1afc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def unfocused_stem(question):   \n",
    "    if not true_or_false(question) or not all_of_the_above(question) or not none_of_the_above(question) or not fill_in_the_blank(question):\n",
    "        return True\n",
    "        \n",
    "    #Traits of an unfocused question (not being a question, etc.)\n",
    "    if '?' not in question.stem and \":\" not in question.stem:\n",
    "        if not question.stem.endswith(('.', ':', '?', ';')):\n",
    "            return False\n",
    "\n",
    "        if not check_if_first_word_is_a_verb(question.stem):\n",
    "            return False\n",
    "        \n",
    "        contains_question = False\n",
    "        doc = nlp(question.stem)\n",
    "        for sent in doc.sents:\n",
    "            if is_question(sent.text.strip()):\n",
    "                contains_question = True\n",
    "                break\n",
    "                \n",
    "        return contains_question\n",
    "\n",
    "    return unfocused_stem_verify(question)\n",
    "\n",
    "def check_if_first_word_is_a_verb(sent):\n",
    "    d = nlp(sent)\n",
    "    token = d[0] # gets the first token in a sentence\n",
    "    if token.pos_ == \"VERB\" and token.dep_ == \"ROOT\": # checks if the first token is a verb and root or not\n",
    "        return True\n",
    "    return False\n",
    "\n",
    "\n",
    "#From https://stackoverflow.com/questions/4083060/determine-if-a-sentence-is-an-inquiry\n",
    "def is_question(sent):\n",
    "    d = nlp(sent)\n",
    "    token = d[0] # gets the first token in a sentence\n",
    "    if token.pos_ == \"VERB\" and token.dep_ == \"ROOT\": # checks if the first token is a verb and root or not\n",
    "        return True\n",
    "    for token in d: # loops through the sentence and checks for WH tokens\n",
    "        if token.tag_ == \"WDT\" or token.tag_ == \"WP\" or token.tag_ == \"WP$\" or token.tag_ == \"WRB\" or token.text == '?':\n",
    "            return True\n",
    "    return  False\n",
    "\n",
    "\n",
    "#Teachers should avoid using MCQs with unfocused stems which do not ask a clear question or state a clear problem in the sentence completion format\n",
    "#The stem should present a clear and focused question that can be understood and answered without looking at the options\n",
    "def unfocused_stem_verify(question):\n",
    "    sysrole = \"\"\"You are an expert, but lenient, educator evaluating multiple-choice questions for the presence of an unfocused stem. A question's stem is considered unfocused if it is not a clear query that can be understood and answered on its own. Your task is to assess how clear and self-contained the stem is, without relying on the answer choices for clarity.\"\"\"\n",
    "    prompt = \"\"\"\n",
    "Instructions:\n",
    "    Provide a single numeric score from 1 to 10, reflecting how focused and self-contained the question’s stem is.\n",
    "    It should ask a clear question or state a clear problem in a sentence completion format.\n",
    "    Ignore questions with options that are similar to \"yes\"/\"no\" or \"true\"/\"false\" followed by an explanation.\n",
    "\n",
    "    Scoring Guidelines:\n",
    "        1–3: The stem is extremely unfocused or unclear, making it difficult for students to understand what is being asked, even after re-reading.\n",
    "        4–6: The stem has noticeable clarity issues; students might need to look at the options for guidance or context.\n",
    "        7–9: The stem is generally clear, with only minor ambiguities; students can understand the question but could benefit from the options for further context.\n",
    "        10: The stem is entirely clear and focused; students can fully understand and answer the question without looking at the options.\n",
    "    \n",
    "    Your response should only be a single number from 1 to 10, with no additional explanation.\n",
    "\n",
    "Question: {}\n",
    "Options: {}\n",
    "Answer: {}\"\"\".format(question.stem, question.options, question.correct_option)\n",
    "\n",
    "    #It's so strict on this, we're setting the threshold to 4 instead of 5.\n",
    "    return llm_rating(sysrole, prompt, 4)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64b015b6-d6bc-4e43-8f04-d82e0c62fc22",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "# Other Metrics (Perplexity, Diversity, Grammatical Error, Cognitive Complexity)\n",
    "In addition to IWF, calculate these other commonly used metrics to see how they evaluate, Answerability is a fifth metric that I am currently leaving out. Just like BLEU, METEOR, ROGUE, etc. these metrics often do not correlate with human judgements and are not indicators of flawed/bad educational multiple-choice questions like the IWF criteria are. You can still compute them because it's easy enough, but you should put little faith in them.\n",
    "\n",
    "### Note, these metrics aren't great indicators, you can read about it here: https://arxiv.org/pdf/2405.20529"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c3f0dc3-0682-4b48-94ed-55653f9fa417",
   "metadata": {},
   "source": [
    "## Perplexity \n",
    "This assesses a language model's ability to predict question and answer text based on its training data. Lower scores suggest more coherent questions and answers with predictable language patterns, whereas higher scores indicate complexity or atypical text, suggesting the questions could be unclear or poorly structured. <br/>\n",
    "<b>NOTE</b>: This will be very slow for a large amount of questions, anything greater than 30 questions will take quite some time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "id": "2a728cf1-0785-4fce-adc9-b22ce5dc7be1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_model(model_id):\n",
    "    tokenizer = GPT2TokenizerFast.from_pretrained(model_id)\n",
    "    model = GPT2LMHeadModel.from_pretrained(model_id)\n",
    "    return tokenizer, model\n",
    "\n",
    "def compute_perplexity(text, tokenizer, model):\n",
    "    inputs = tokenizer(text, return_tensors='pt')\n",
    "    with torch.no_grad():\n",
    "        outputs = model(**inputs, labels=inputs['input_ids'])\n",
    "    loss = outputs.loss\n",
    "    perplexity = torch.exp(loss)\n",
    "    return perplexity.item()\n",
    "\n",
    "def perplexity(questions, model_id='gpt2-large'):\n",
    "    tokenizer, model = load_model(model_id)\n",
    "    nl = ', '\n",
    "    perplexities = []\n",
    "    \n",
    "    for index, row in questions.iterrows():\n",
    "        stem = row['text'].strip()\n",
    "        non_empty_values = [row[col].strip() for col in ['a', 'b', 'c', 'd', 'e'] if row[col].strip()]\n",
    "        row_string = nl.join(non_empty_values)\n",
    "        text = stem + ' ' + row_string\n",
    "        perplexity_value = compute_perplexity(text, tokenizer, model)\n",
    "        perplexities.append(perplexity_value)\n",
    "    \n",
    "    return perplexities"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "905d96ac-e318-4fdf-b716-a00f7ed5ae98",
   "metadata": {},
   "source": [
    "## Diversity\n",
    "Using Distinct-3, this evaluates the range in vocabulary, structure, and content across generated texts, ensuring a variety of questions and answers and reducing repetition. A higher diversity score indicates greater uniqueness among MCQs, avoiding repetitive phrases and templated patterns. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "id": "31dc2604-dc17-4200-8507-f11cdd68af34",
   "metadata": {},
   "outputs": [],
   "source": [
    "def diversity(questions):\n",
    "    predictions = []\n",
    "    per_question = []\n",
    "    for index, row in questions.iterrows():\n",
    "        stem=row['text'].strip()\n",
    "        non_empty_values = [row[col].strip() for col in ['a','b','c','d'] if row[col].strip()]\n",
    "        row_string = ', '.join(non_empty_values)\n",
    "\n",
    "        predictions.append(stem + ' ' + row_string)\n",
    "    \n",
    "    distinct_3_total = 0\n",
    "    for o in predictions:\n",
    "        dist3 = calculate_distinct_3(o)\n",
    "        distinct_3_total = distinct_3_total + dist3\n",
    "        per_question.append(dist3)\n",
    "\n",
    "    print('distinct_3_total: ', (distinct_3_total)/len(predictions))   \n",
    "    print('ngram_diversity_total: ', ngram_diversity(predictions))\n",
    "    print('length: ', len(predictions))\n",
    "\n",
    "    return per_question\n",
    "\n",
    "def calculate_distinct_3(text):\n",
    "    # Tokenize the text\n",
    "    tokens = word_tokenize(text)\n",
    "\n",
    "    # Generate trigrams\n",
    "    trigrams_list = list(trigrams(tokens))\n",
    "\n",
    "    # Count unique trigrams\n",
    "    unique_trigrams = set(trigrams_list)\n",
    "    num_unique_trigrams = len(unique_trigrams)\n",
    "\n",
    "    # Count total trigrams\n",
    "    total_trigrams = len(trigrams_list)\n",
    "\n",
    "    # Calculate Distinct-3\n",
    "    if total_trigrams > 0:\n",
    "        distinct_3 = num_unique_trigrams / total_trigrams\n",
    "    else:\n",
    "        distinct_3 = 0\n",
    "\n",
    "    return distinct_3\n",
    "\n",
    "def ngram_diversity(options, n=3):\n",
    "    all_ngrams = [ngram for option in options for ngram in ngrams(word_tokenize(option), n)]\n",
    "    unique_ngrams = set(all_ngrams)\n",
    "    return len(unique_ngrams) / len(all_ngrams)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "015d5690-30cd-40b0-b22b-0822aecae055",
   "metadata": {},
   "source": [
    "## Grammatical Error\n",
    "This uses a Python wrapper for https://languagetool.org/ currently we are using the free API endpoint, so if it's used excessively we might get IP blocked. Grammatical errors pinpoint grammar violations, such as incorrect verb tense or spelling, quantified for each MCQ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "id": "6fcf1983-b487-49c0-9fc7-4ca4436f4341",
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_grammar(text):\n",
    "    tool = language_tool_python.LanguageToolPublicAPI('en-US')\n",
    "    matches = tool.check(text)\n",
    "    return len(matches), matches\n",
    "\n",
    "predictions = []\n",
    "ques = {}\n",
    "\n",
    "def grammatical_error(questions):\n",
    "    total_errors = 0\n",
    "    errorsList = []\n",
    "    for index, row in questions.iterrows():\n",
    "        stem=row['text'].strip()\n",
    "        num_errors, errors = check_grammar(stem)\n",
    "        total_errors = total_errors + num_errors\n",
    "        errorsList.append(num_errors)\n",
    "\n",
    "    print('total_errors: ', (total_errors/len(questions)))   \n",
    "    print('length: ', len(questions))\n",
    "    print('length of errors: ', len(errorsList))\n",
    "\n",
    "    return errorsList"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "057a3db5-0fbb-4a24-9089-d2a66f0cab27",
   "metadata": {},
   "source": [
    "## Cognitive Complexity\n",
    "This is measured by Bloom's Taxonomy, although some research has done it by the \"difficulty\" of the question, which a LLM can assess, but Bloom's is a better fit. Additionally, this might be redundant since the Bloom's label is included in the question's construction.plex\n",
    "\n",
    "### Note this is only set up for MCQs with 4 optons at the moment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "id": "f9d0582d-0993-450f-bad4-25a629b74921",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cognitive_complexity(questions):\n",
    "    bloom_labels = []\n",
    "    predictions = []\n",
    "    for index, row in questions.iterrows():\n",
    "        stem=row['text'].strip()\n",
    "        non_empty_values = [row[col].strip() for col in ['a','b','c','d', 'e'] if row[col].strip()]\n",
    "        row_string = '\\n'.join(non_empty_values)\n",
    "        predictions.append(stem + '\\n' + row_string)\n",
    "\n",
    "    sysrole = \"You are an expert in pedagogy and an astute instructor here to classify a multiple-choice questions provided to you with one of the six level's of Bloom's Revised Taxonomy\"\n",
    "    prompt = \"\"\"Given the multiple-choice question below, please respond with that level of Bloom's Revised Taxonomy it falls into and nothing else.\n",
    "        {}\n",
    "        \"\"\"\n",
    "    for q in predictions:\n",
    "        p = prompt.format(q)\n",
    "        done = False\n",
    "    \n",
    "        while(done == False):\n",
    "            try:\n",
    "                o = openai.chat.completions.create(\n",
    "                  model=model_engine,\n",
    "                  messages=[\n",
    "                     {\"role\": \"system\",\n",
    "                      \"content\": sysrole},\n",
    "                    {\"role\": \"user\", \n",
    "                     \"content\": p},\n",
    "                  ],\n",
    "                  max_tokens = 4096,\n",
    "                  temperature = 0.7\n",
    "                 )\n",
    "                done = True \n",
    "            except Exception as error:\n",
    "                print('errored in LLM API call: ', error)\n",
    "                time.sleep(10)\n",
    "        completion = o\n",
    "        done = False\n",
    "    \n",
    "        try:\n",
    "            expert_reasoning = completion.choices[0].message.content.lower()\n",
    "            bloom_labels.append(expert_reasoning)\n",
    "        except: \n",
    "            print('error with LLM: ', completion)\n",
    "    return bloom_labels"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55054a9c-ebe5-40e3-be58-d643f7af1cf6",
   "metadata": {},
   "source": [
    "# Formatting Your CSV of MCQs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f526d878-c06d-4016-aadc-98705e86c768",
   "metadata": {},
   "source": [
    "| id | text | answer | a | b | c | d |\r\n",
    "|----------|----------|----------|----------|----------|----------|----------|\r\n",
    "| Data 1   | Data 2   | Data 3   | Data 4   | Data 5   | Data 6   | Data 7   |\r\n",
    "| Data 8   | Data 9   | Data 10  | Data 11  | Data 12  | Data 13  | Data 14 |\r\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ece7db81-8d41-4145-aad5-b73257b8c6f9",
   "metadata": {},
   "source": [
    "### id: A unique number\n",
    "### text: The question's stem\n",
    "### answer: The text of the correct response, this should match the text in one of the options (a-h) columns\n",
    "### a-h: The text for the corresponding option (the code now works for any amount of distractors up to 8 in total!)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28b715b5-4e7c-4e4d-be17-f2873c902202",
   "metadata": {},
   "source": [
    "# 19 Item-Writing Flaws Criteria\n",
    "# This code works for MCQs with 2-8 options (a-h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "id": "49ea0746-d79a-4887-a645-208ad62a62a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing file: benchmark_no_iwfs/biochemistry.csv\n",
      "Applying criteria: complex_k_type\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['1', '1']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['8', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "Saved output to: benchmark_no_iwfs/biochemistry_4o_comp.csv\n",
      "Processing file: benchmark_no_iwfs/chemistry.csv\n",
      "Applying criteria: complex_k_type\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['1', '1']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['1', '1']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "Saved output to: benchmark_no_iwfs/chemistry_4o_comp.csv\n",
      "Processing file: benchmark_no_iwfs/computer_science.csv\n",
      "Applying criteria: complex_k_type\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['1', '3']\n",
      "ratings:  ['1', '1']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '7']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['4', '1']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['1', '1']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['4', '10', '7']\n",
      "ratings:  ['10', '7']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "Saved output to: benchmark_no_iwfs/computer_science_4o_comp.csv\n",
      "Processing file: benchmark_no_iwfs/healthcare.csv\n",
      "Applying criteria: complex_k_type\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['1', '1']\n",
      "ratings:  ['1', '1']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "Saved output to: benchmark_no_iwfs/healthcare_4o_comp.csv\n",
      "Processing file: benchmark_no_iwfs/humanities.csv\n",
      "Applying criteria: complex_k_type\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "Saved output to: benchmark_no_iwfs/humanities_4o_comp.csv\n",
      "Processing file: benchmark_no_iwfs/statistics.csv\n",
      "Applying criteria: complex_k_type\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "ratings:  ['10', '10']\n",
      "Saved output to: benchmark_no_iwfs/statistics_4o_comp.csv\n"
     ]
    }
   ],
   "source": [
    "all_criteria = [\n",
    "    'ambiguous_unclear_information',\n",
    "    'implausible_distractors',\n",
    "    'none_of_the_above',\n",
    "    'longest_answer_correct',\n",
    "    'gratuitous_information_in_stem',\n",
    "    'true_or_false',\n",
    "    'avoid_convergence_cues',\n",
    "    'avoid_logical_cues',\n",
    "    'all_of_the_above',\n",
    "    'fill_in_the_blank',\n",
    "    'absolute_terms',\n",
    "    'word_repeats_in_stem_and_correct_answer',\n",
    "    'unfocused_stem',\n",
    "    'complex_k_type',\n",
    "    'grammatical_cues_in_stem',\n",
    "    'lost_sequence',\n",
    "    'vague_terms',\n",
    "    'more_than_one_correct',\n",
    "    'negative_worded_stem'\n",
    "]\n",
    "\n",
    "# List all your CSV file paths here\n",
    "files = [\n",
    "    'yourPath/YouFileWithTheCorrectColumns.csv',\n",
    "    #More files if needed can go here\n",
    "]\n",
    "\n",
    "# Loop over each file individually\n",
    "for file in files:\n",
    "    print(f'Processing file: {file}')\n",
    "    \n",
    "    # Initialize data structures for each file\n",
    "    all_data = {}\n",
    "    questions = []\n",
    "\n",
    "    # Read the CSV file\n",
    "    data = pd.read_csv(file).fillna('')\n",
    "    \n",
    "    #works for MCQs with up to 8 options\n",
    "    options_columns = [col for col in data.columns if col not in ['id', 'text', 'answer']]\n",
    "    option_labels = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'][:len(options_columns)]\n",
    "    \n",
    "    for index, row in data.iterrows():\n",
    "        options = [row[col].strip() for col in options_columns if row[col].strip()]\n",
    "        question = MultipleChoiceQuestion(\n",
    "            stem=row['text'],\n",
    "            options=options,\n",
    "            correct_option=row['answer'].strip(),\n",
    "            qid=row['id'],\n",
    "            quality=0\n",
    "        )\n",
    "        questions.append(question)\n",
    "\n",
    "    # Apply each criterion to the questions\n",
    "    for criteria in all_criteria:\n",
    "        print(f'Applying criteria: {criteria}')\n",
    "        auto_iwf_results = [0 if globals()[criteria](q) else 1 for q in questions]\n",
    "        all_data[criteria] = auto_iwf_results\n",
    "\n",
    "    def pad_row(row, length=len(options_columns) + 3, pad_value=''):\n",
    "        return row + [pad_value] * (length - len(row))\n",
    "        \n",
    "    padded_questions = [pad_row([q.qid, q.stem, q.correct_option] + q.options) for q in questions]\n",
    "    headers = ['id', 'text', 'answer'] + option_labels\n",
    "    qdf = pd.DataFrame(padded_questions, columns=headers)\n",
    "    df = pd.DataFrame(all_data)\n",
    "    combined_df = pd.concat([qdf, df], axis=1)\n",
    "\n",
    "    output_filename = file.replace('.csv', '_IWFresults.csv')\n",
    "    combined_df.to_csv('results/' + output_filename, index=False)\n",
    "    print(f'Saved output to: {output_filename}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "088786ef-9462-4f99-a90d-b80867fb7b62",
   "metadata": {},
   "source": [
    "# Other Metrics (Perplexity, Diversity, Grammatical Error, Cognitive Complexity) - Running the code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab3fc9b7-6b81-4557-9f0e-e3d66860d476",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "t = erl letting it error\n",
    "## Calc the other metrics\n",
    "other_metrics = ['perplexity',\n",
    "                 'diversity',\n",
    "                 'grammatical_error',\n",
    "                 'cognitive_complexity']\n",
    "\n",
    "files = ['CSV_OTHER.csv']\n",
    "metric_data = {}\n",
    "for metric in other_metrics:\n",
    "    for file in files:\n",
    "        print('----- ', metric, ' ----- ', file)\n",
    "        data = pd.read_csv(file)\n",
    "        data = data.fillna('')\n",
    "        combined_data = pd.concat([data]) #This is used for multiple files\n",
    "        \n",
    "        result = globals()[metric](combined_data)\n",
    "        metric_data[metric] = result\n",
    "\n",
    "df = pd.DataFrame(metric_data)\n",
    "df.to_csv('RESULTS_OTHER.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}