eva_baseline_without_language_models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e24928a4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\z004r5cc\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "\n",
    "from gpt_ner_api_codes.prompts_all import *\n",
    "from gpt_ner_api_codes.demonstration_samples_generation import *\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import tqdm\n",
    "def nop(it, *a, **k):\n",
    "    return itm\n",
    "tqdm.tqdm = nop   \n",
    "\n",
    "from simcse import SimCSE\n",
    "model = SimCSE(\"princeton-nlp/sup-simcse-bert-base-uncased\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0cd7c97a",
   "metadata": {},
   "source": [
    "## Thin film head technology dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "8ce67ace",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array(['AnnotatorNotes', 'Attribution', 'Component', 'Consequence',\n",
      "       'Effect', 'EnergyFlow', 'Function', 'InfoFlow', 'Location',\n",
      "       'Material', 'Measure', 'PhysicsFlow', 'ScientificConcept', 'Shape',\n",
      "       'State', 'System', 'Value'], dtype='<U1679'), array([   25,  1908, 12911,   132,   750,  1573,  1542,   203,  2601,\n",
      "        1683,   152,   147,   711,  1074,    40,  1134,   312],\n",
      "      dtype=int64))\n",
      "Duplicated few-shot examples:  306\n",
      "length of stratified_sentences_develop:  283\n",
      "length of stratified_sentences_test:  500\n",
      "length of stratified_sentences_train:  3760\n"
     ]
    }
   ],
   "source": [
    "path_original_sentence = \"./data/thin-film-technology-dataset/thin_film_head_technology_total_original_sentence.jsonl\"\n",
    "path_entities = \"./data/thin-film-technology-dataset/thin_film_head_technology_total_entities.jsonl\"\n",
    "path_tokens_tags = \"./data/thin-film-technology-dataset/thin_film_head_technology_total.jsonl\"\n",
    "sentences_dev, sentences_input, sentences_train, demonstrations_dev, demonstrations_input_solutions, demonstrations_train = split_dev_dataset_input(path_original_sentence, path_entities, path_tokens_tags, num_input = 500, demon_size = 300, stratify = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0139f847",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a55e5a15",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "500 273\n",
      "TiW film: Component\n",
      "3 nm to 5 nm: Value\n",
      "thickness: Attribution\n",
      "base film: Component\n",
      "bias magnet film: Component\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "with open(\"data/immutable_data_formal/sen_thf_input_30.txt\", \"rb\") as data:   #Pickling\n",
    "    sentences_input = pickle.load(data)  \n",
    "with open(\"data/immutable_data_formal/dem_thf_input_30.txt\", \"rb\") as data:   #Pickling\n",
    "    demonstrations_input_solutions = pickle.load(data) \n",
    "print(len(sentences_input), len(demonstrations_dev) )\n",
    "print(demonstrations_dev[2])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d9ddb9c4",
   "metadata": {},
   "source": [
    "#### Long list of words and corresponding entities in development dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a75c823c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13979 ['(', '1', ')', 'A', 'magnetic', 'recording', 'system', 'is', 'disclosed', ',']\n",
      "13979 ['0', '0', '0', '0', 'System', 'System', 'System', '0', '0', '0']\n"
     ]
    }
   ],
   "source": [
    "words_dev = []\n",
    "words_entities_dev = []\n",
    "for i, sentence in enumerate(sentences_dev):\n",
    "    for token_index, token in enumerate(sentence.split(\" \")):\n",
    "        words_dev.append(token)\n",
    "        #get the list of entities for each sentence\n",
    "        for demon_row in demonstrations_dev[i].split(\"\\n\"):#### Long list of words and corresponding entities in development dataset\n",
    "            if token in demon_row.split(\": \")[0]:\n",
    "                words_entities_dev.append(demon_row.split(\": \")[1])\n",
    "                break\n",
    "        if len(words_dev) == len(words_entities_dev)+1:\n",
    "                words_entities_dev.append(\"0\")  \n",
    "print(len(words_dev), words_dev[0:10])\n",
    "print(len(words_entities_dev), words_entities_dev[0:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc2914d7",
   "metadata": {},
   "source": [
    "#### Long list of words and corresponding entities in input sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7c769731",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17580\n"
     ]
    }
   ],
   "source": [
    "words_input_sentence = []\n",
    "for sentence in sentences_input:\n",
    "    #print(sentence)\n",
    "    for token in sentence.split(\" \"):\n",
    "        words_input_sentence.append(token)\n",
    "print(len(words_input_sentence))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "37b04f82",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 275/275 [02:40<00:00,  1.72it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 219/219 [02:18<00:00,  1.58it/s]\n"
     ]
    }
   ],
   "source": [
    "similarity_matrix = model.similarity(words_input_sentence, words_dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4d47f724",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['0' '0' '0' ... '0' 'EnergyFlow' '0']\n",
      " ['Location' 'Component' 'Component' ... 'Component' 'EnergyFlow'\n",
      "  'Component']\n",
      " ['Component' 'Component' 'Component' ... 'ScientificConcept' 'Component'\n",
      "  '0']\n",
      " ...\n",
      " ['Component' 'ScientificConcept' 'Component' ... 'ScientificConcept'\n",
      "  'Effect' 'ScientificConcept']\n",
      " ['0' '0' '0' ... '0' '0' '0']\n",
      " ['0' '0' '0' ... '0' '0' '0']]\n",
      "17580\n",
      "17580\n"
     ]
    }
   ],
   "source": [
    "top_index_xb = np.argsort( -similarity_matrix, axis=1)\n",
    "words_dev_array = np.asarray(words_dev)\n",
    "words_entities_dev_array = np.asarray(words_entities_dev)\n",
    "words_dev_array[top_index_xb[:,:50]]\n",
    "ranked_entities = words_entities_dev_array[top_index_xb[:,:50]]\n",
    "print(ranked_entities)\n",
    "\n",
    "def most_frequent(List):\n",
    "    return max(set(List), key = List.count)\n",
    "ranked_entities = list(ranked_entities)\n",
    "most_frequent_entities = []\n",
    "for ranked_entities_one_token in ranked_entities:\n",
    "    most_frequent_entities.append( most_frequent( list(ranked_entities_one_token)) )\n",
    "print(len(words_input_sentence) )\n",
    "print(len(most_frequent_entities)) \n",
    "#print(most_frequent_entities)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "98e427a4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pole: Component\n",
      "pieces: Component\n",
      "or: Component\n",
      "core: Component\n",
      "thin: Component\n",
      "film: Component\n",
      "head: Component\n",
      "electrically: Component\n",
      "base: EnergyFlow\n",
      "substrate: Component\n",
      "thin: Component\n",
      "film: Component\n",
      "head: Component\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# get the output format like gpt \n",
    "i = 0\n",
    "response = []\n",
    "for sentence_index, sentence in enumerate(sentences_input):\n",
    "    response_one_sentence = \"\"\n",
    "    for token in sentence.split(\" \"):\n",
    "        if most_frequent_entities[i] == \"0\":\n",
    "            i = i + 1\n",
    "            continue\n",
    "        else:\n",
    "            response_one_sentence = response_one_sentence + token + \": \" + most_frequent_entities[i] + \"\\n\"\n",
    "        i = i + 1\n",
    "    response.append( response_one_sentence + \"\\n\")    \n",
    "print(response[0] ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "cac71d4c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['System', 'AnnotatorNotes', 'Material', 'Value', 'ScientificConcept', 'Consequence', 'Shape', 'Location', 'Measure', 'Effect', 'PhysicsFlow', 'InfoFlow', 'Function', 'EnergyFlow', 'Component', 'Attribution', 'State']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entity</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>F1 score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>System</td>\n",
       "      <td>39.43662</td>\n",
       "      <td>11.522634</td>\n",
       "      <td>0.178344</td>\n",
       "      <td>243.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AnnotatorNotes</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Material</td>\n",
       "      <td>45.180723</td>\n",
       "      <td>23.734177</td>\n",
       "      <td>0.311203</td>\n",
       "      <td>316.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Value</td>\n",
       "      <td>41.37931</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.269663</td>\n",
       "      <td>60.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ScientificConcept</td>\n",
       "      <td>18.518519</td>\n",
       "      <td>2.747253</td>\n",
       "      <td>0.047847</td>\n",
       "      <td>182.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Consequence</td>\n",
       "      <td>33.333333</td>\n",
       "      <td>4.545455</td>\n",
       "      <td>0.08</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Shape</td>\n",
       "      <td>46.808511</td>\n",
       "      <td>11.518325</td>\n",
       "      <td>0.184874</td>\n",
       "      <td>191.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Location</td>\n",
       "      <td>48.375451</td>\n",
       "      <td>28.270042</td>\n",
       "      <td>0.356858</td>\n",
       "      <td>474.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Measure</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>38.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Effect</td>\n",
       "      <td>27.142857</td>\n",
       "      <td>7.279693</td>\n",
       "      <td>0.114804</td>\n",
       "      <td>261.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>PhysicsFlow</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>28.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>InfoFlow</td>\n",
       "      <td>43.103448</td>\n",
       "      <td>28.089888</td>\n",
       "      <td>0.340136</td>\n",
       "      <td>89.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Function</td>\n",
       "      <td>51.351351</td>\n",
       "      <td>6.050955</td>\n",
       "      <td>0.108262</td>\n",
       "      <td>314.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>EnergyFlow</td>\n",
       "      <td>45.238095</td>\n",
       "      <td>14.921466</td>\n",
       "      <td>0.224409</td>\n",
       "      <td>382.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Component</td>\n",
       "      <td>53.207547</td>\n",
       "      <td>51.272727</td>\n",
       "      <td>0.522222</td>\n",
       "      <td>2200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Attribution</td>\n",
       "      <td>39.380531</td>\n",
       "      <td>23.924731</td>\n",
       "      <td>0.297659</td>\n",
       "      <td>372.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>State</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>accuracy</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>31.152287</td>\n",
       "      <td>5181.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>macro avg</td>\n",
       "      <td>31.320959</td>\n",
       "      <td>13.757491</td>\n",
       "      <td>0.178605</td>\n",
       "      <td>5181.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>weighted avg</td>\n",
       "      <td>46.004437</td>\n",
       "      <td>31.152287</td>\n",
       "      <td>0.349809</td>\n",
       "      <td>5181.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               entity  precision     recall   F1 score  support\n",
       "0              System   39.43662  11.522634   0.178344    243.0\n",
       "1      AnnotatorNotes        0.0        0.0        0.0      3.0\n",
       "2            Material  45.180723  23.734177   0.311203    316.0\n",
       "3               Value   41.37931       20.0   0.269663     60.0\n",
       "4   ScientificConcept  18.518519   2.747253   0.047847    182.0\n",
       "5         Consequence  33.333333   4.545455       0.08     22.0\n",
       "6               Shape  46.808511  11.518325   0.184874    191.0\n",
       "7            Location  48.375451  28.270042   0.356858    474.0\n",
       "8             Measure        0.0        0.0        0.0     38.0\n",
       "9              Effect  27.142857   7.279693   0.114804    261.0\n",
       "10        PhysicsFlow        0.0        0.0        0.0     28.0\n",
       "11           InfoFlow  43.103448  28.089888   0.340136     89.0\n",
       "12           Function  51.351351   6.050955   0.108262    314.0\n",
       "13         EnergyFlow  45.238095  14.921466   0.224409    382.0\n",
       "14          Component  53.207547  51.272727   0.522222   2200.0\n",
       "15        Attribution  39.380531  23.924731   0.297659    372.0\n",
       "16              State        0.0        0.0        0.0      6.0\n",
       "0            accuracy          -          -  31.152287   5181.0\n",
       "1           macro avg  31.320959  13.757491   0.178605   5181.0\n",
       "2        weighted avg  46.004437  31.152287   0.349809   5181.0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transformed_output = change_output_format_to_tokens_tags(sentences_input, response)\n",
    "transformed_solution = change_output_format_to_tokens_tags(sentences_input, demonstrations_input_solutions)\n",
    "get_evaluation_without_o(transformed_solution = transformed_solution, transformed_output = transformed_output) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "caecd50b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "0b0930ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "def eva_baseline( path_sen_train, path_dem_train, path_sen_input, path_dem_input ):\n",
    "        \n",
    "    with open(path_sen_train, \"rb\") as data:   #Pickling\n",
    "        sentences_dev = pickle.load(data)  \n",
    "    with open(path_dem_train, \"rb\") as data:   #Pickling\n",
    "        demonstrations_dev = pickle.load(data)     \n",
    "    \n",
    "    with open(path_sen_input, \"rb\") as data:   #Pickling\n",
    "        sentences_input = pickle.load(data)  \n",
    "    with open(path_dem_input, \"rb\") as data:   #Pickling\n",
    "        demonstrations_input_solutions = pickle.load(data) \n",
    "    print(len(sentences_input), len(demonstrations_dev) )\n",
    "    print(demonstrations_dev[2])\n",
    "    \n",
    "    ########################################\n",
    "    # long list of words and entities\n",
    "    ########################################    \n",
    "    words_dev = []\n",
    "    words_entities_dev = []\n",
    "    for i, sentence in enumerate(sentences_dev):\n",
    "        for token_index, token in enumerate(sentence.split(\" \")):\n",
    "            words_dev.append(token)\n",
    "            #get the list of entities for each sentence\n",
    "            for demon_row in demonstrations_dev[i].split(\"\\n\"):#### Long list of words and corresponding entities in development dataset\n",
    "                if token in demon_row.split(\": \")[0]:\n",
    "                    words_entities_dev.append(demon_row.split(\": \")[1])\n",
    "                    break\n",
    "            if len(words_dev) == len(words_entities_dev)+1:\n",
    "                    words_entities_dev.append(\"0\")  \n",
    "    print(len(words_dev), words_dev[0:10])\n",
    "    print(len(words_entities_dev), words_entities_dev[0:10])\n",
    "    \n",
    "    ########################################\n",
    "    # Long list of words and corresponding entities in input sentence\n",
    "    ########################################  \n",
    "    # words_input_sentence = []\n",
    "    for sentence in sentences_input:\n",
    "        #print(sentence)\n",
    "        for token in sentence.split(\" \"):\n",
    "            words_input_sentence.append(token)\n",
    "    print(len(words_input_sentence)) \n",
    "    \n",
    "    \n",
    "    ############## use CSE to measure similarities\n",
    "    similarity_matrix = model.similarity(words_input_sentence, words_dev)\n",
    "    \n",
    "    top_index_xb = np.argsort( -similarity_matrix, axis=1)\n",
    "    words_dev_array = np.asarray(words_dev)\n",
    "    words_entities_dev_array = np.asarray(words_entities_dev)\n",
    "    words_dev_array[top_index_xb[:,:50]]\n",
    "    ranked_entities = words_entities_dev_array[top_index_xb[:,:50]]\n",
    "    #print(ranked_entities)\n",
    "\n",
    "    def most_frequent(List):\n",
    "        return max(set(List), key = List.count)\n",
    "    ranked_entities = list(ranked_entities)\n",
    "    most_frequent_entities = []\n",
    "    for ranked_entities_one_token in ranked_entities:\n",
    "        most_frequent_entities.append( most_frequent( list(ranked_entities_one_token)) )\n",
    "    print(len(words_input_sentence) )\n",
    "    print(len(most_frequent_entities)) \n",
    "    #print(most_frequent_entities)\n",
    "    \n",
    "\n",
    "    ########################################\n",
    "    # get the output format like gpt \n",
    "    ########################################      \n",
    "    i = 0\n",
    "    response = []\n",
    "    for sentence_index, sentence in enumerate(sentences_input):\n",
    "        response_one_sentence = \"\"\n",
    "        for token in sentence.split(\" \"):\n",
    "            if most_frequent_entities[i] == \"0\":\n",
    "                i = i + 1\n",
    "                continue\n",
    "            else:\n",
    "                response_one_sentence = response_one_sentence + token + \": \" + most_frequent_entities[i] + \"\\n\"\n",
    "            i = i + 1\n",
    "        response.append( response_one_sentence + \"\\n\")    \n",
    "    #print(response[0] ) \n",
    "    \n",
    "    ########## evaluation\n",
    "    transformed_output = change_output_format_to_tokens_tags(sentences_input, response)\n",
    "    transformed_solution = change_output_format_to_tokens_tags(sentences_input, demonstrations_input_solutions)\n",
    "    get_evaluation_without_o(transformed_solution = transformed_solution, transformed_output = transformed_output) \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "0f6c5592",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "500 3760\n",
      "magnetic recording system: Component\n",
      "thin-film magnetic head: Component\n",
      "magnetic recording medium: Material\n",
      "nonmagnetic support: Component\n",
      "nonmagnetic layer: Component\n",
      "inorganic nonmagnetic particles: Material\n",
      "binder resin: PhysicsFlow\n",
      "nonmagnetic support: Component\n",
      "magnetic layer: Component\n",
      "mainly ferromagnetic metal particles: Material\n",
      "binder: PhysicsFlow\n",
      "resin: Material\n",
      "nonmagnetic layer: Component\n",
      "binder resins: PhysicsFlow\n",
      "magnetic and nonmagnetic layers: Component\n",
      "vinyl chloride resin: PhysicsFlow\n",
      "ferromagnetic metal particles: Material\n",
      "organic compound: Material\n",
      "magnetic layer: Component\n",
      "thickness: Attribution\n",
      "0.05 to 1.0: Value\n",
      "\n",
      "\n",
      "130029 [\"''\", ')', 'is', 'then', 'deposited', 'over', 'the', 'MSL', 'layer', 'and']\n",
      "130029 ['0', '0', '0', '0', '0', '0', '0', 'Component', 'Component', '0']\n",
      "49641\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████| 776/776 [05:42<00:00,  2.27it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████| 2032/2032 [17:52<00:00,  1.89it/s]\n"
     ]
    },
    {
     "ename": "MemoryError",
     "evalue": "Unable to allocate 48.1 GiB for an array with shape (49641, 130029) and data type int64",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_36732\\2302015084.py\u001b[0m in \u001b[0;36m<cell line: 5>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mpath_sen_input\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"data/immutable_data_formal/sen_thf_input_30.txt\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mpath_dem_input\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"data/immutable_data_formal/dem_thf_input_30.txt\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m \u001b[0meva_baseline\u001b[0m\u001b[1;33m(\u001b[0m \u001b[0mpath_sen_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpath_dem_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpath_sen_input\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpath_dem_input\u001b[0m \u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_36732\\470774015.py\u001b[0m in \u001b[0;36meva_baseline\u001b[1;34m(path_sen_train, path_dem_train, path_sen_input, path_dem_input)\u001b[0m\n\u001b[0;32m     45\u001b[0m     \u001b[0msimilarity_matrix\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msimilarity\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwords_input_sentence\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mwords_dev\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     46\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 47\u001b[1;33m     \u001b[0mtop_index_xb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0margsort\u001b[0m\u001b[1;33m(\u001b[0m \u001b[1;33m-\u001b[0m\u001b[0msimilarity_matrix\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     48\u001b[0m     \u001b[0mwords_dev_array\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwords_dev\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     49\u001b[0m     \u001b[0mwords_entities_dev_array\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwords_entities_dev\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\siemens old windows machine document folder\\master_thesis_code\\instructner0829\\instructner\\.gptdata_hf38\\lib\\site-packages\\numpy\\core\\overrides.py\u001b[0m in \u001b[0;36margsort\u001b[1;34m(*args, **kwargs)\u001b[0m\n",
      "\u001b[1;32mc:\\siemens old windows machine document folder\\master_thesis_code\\instructner0829\\instructner\\.gptdata_hf38\\lib\\site-packages\\numpy\\core\\fromnumeric.py\u001b[0m in \u001b[0;36margsort\u001b[1;34m(a, axis, kind, order)\u001b[0m\n\u001b[0;32m   1144\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1145\u001b[0m     \"\"\"\n\u001b[1;32m-> 1146\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0m_wrapfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'argsort'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkind\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mkind\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1147\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1148\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\siemens old windows machine document folder\\master_thesis_code\\instructner0829\\instructner\\.gptdata_hf38\\lib\\site-packages\\numpy\\core\\fromnumeric.py\u001b[0m in \u001b[0;36m_wrapfunc\u001b[1;34m(obj, method, *args, **kwds)\u001b[0m\n\u001b[0;32m     55\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     56\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 57\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mbound\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     58\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     59\u001b[0m         \u001b[1;31m# A TypeError occurs if the object does have such a method in its\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mMemoryError\u001b[0m: Unable to allocate 48.1 GiB for an array with shape (49641, 130029) and data type int64"
     ]
    }
   ],
   "source": [
    "path_sen_train = \"data/immutable_data_formal/sen_thf_train.txt\"\n",
    "path_dem_train = \"data/immutable_data_formal/dem_thf_train.txt\"\n",
    "path_sen_input = \"data/immutable_data_formal/sen_thf_input_30.txt\"\n",
    "path_dem_input = \"data/immutable_data_formal/dem_thf_input_30.txt\"\n",
    "eva_baseline( path_sen_train, path_dem_train, path_sen_input, path_dem_input )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fcbbbde3",
   "metadata": {},
   "source": [
    "# Assembly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "1793da77",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array(['DIM', 'ID', 'OPER', 'PART', 'PROP', 'QTY', 'RPOS', 'TOOL', 'WGT'],\n",
      "      dtype='<U650'), array([  55,  101,  735, 2528,   84,   92,  699,  108,    2], dtype=int64))\n",
      "Duplicated few-shot examples:  300\n",
      "length of stratified_sentences_develop:  239\n",
      "length of stratified_sentences_test:  399\n",
      "length of stratified_sentences_train:  776\n"
     ]
    }
   ],
   "source": [
    "path_original_sentence = \"./data/assembly_dataset/alternators-engines-gearboxes/train_original_sentence.jsonl\"\n",
    "path_entities = \"./data/assembly_dataset/alternators-engines-gearboxes/train_entities.jsonl\"\n",
    "path_tokens_tags = \"./data/assembly_dataset/alternators-engines-gearboxes/train.jsonl\"\n",
    "sentences_dev, sentences_input, sentences_train, demonstrations_dev, demonstrations_input_solutions, demonstrations_train = split_dev_dataset_input(path_original_sentence, path_entities, path_tokens_tags, num_input = 500, demon_size = 300, stratify = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "ffef2874",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/immutable_data_formal/sen_assembly_input_30.txt\", \"rb\") as data:   #Pickling\n",
    "    sentences_input = pickle.load(data)  \n",
    "with open(\"data/immutable_data_formal/dem_assembly_input_30.txt\", \"rb\") as data:   #Pickling\n",
    "    demonstrations_input_solutions = pickle.load(data)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "e407bd0c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7319 ['#', 'Install', 'guide', '0', '\\n', '&', 'into', 'exhaust', 'rear', '769938']\n",
      "7319 ['ID', 'OPER', '0', '0', '0', '0', 'RPOS', 'PART', 'RPOS', 'ID']\n"
     ]
    }
   ],
   "source": [
    "words_dev = []\n",
    "words_entities_dev = []\n",
    "for i, sentence in enumerate(sentences_dev):\n",
    "    for token_index, token in enumerate(sentence.split(\" \")):\n",
    "        words_dev.append(token)\n",
    "        #get the list of entities for each sentence\n",
    "        for demon_row in demonstrations_dev[i].split(\"\\n\"):#### Long list of words and corresponding entities in development dataset\n",
    "            if token in demon_row.split(\": \")[0]:\n",
    "                words_entities_dev.append(demon_row.split(\": \")[1])\n",
    "                break\n",
    "        if len(words_dev) == len(words_entities_dev)+1:\n",
    "                words_entities_dev.append(\"0\")  \n",
    "print(len(words_dev), words_dev[0:10])\n",
    "print(len(words_entities_dev), words_entities_dev[0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "a76338b3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11190\n"
     ]
    }
   ],
   "source": [
    "words_input_sentence = []\n",
    "for sentence in sentences_input:\n",
    "    #print(sentence)\n",
    "    for token in sentence.split(\" \"):\n",
    "        words_input_sentence.append(token)\n",
    "print(len(words_input_sentence))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "4cbe1ee5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 175/175 [01:50<00:00,  1.58it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:11<00:00,  1.61it/s]\n"
     ]
    }
   ],
   "source": [
    "similarity_matrix = model.similarity(words_input_sentence, words_dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "04dba315",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['ID' '0' 'ID' ... '0' '0' '0']\n",
      " ['0' '0' '0' ... '0' '0' '0']\n",
      " ['0' '0' '0' ... '0' '0' '0']\n",
      " ...\n",
      " ['PROP' 'PROP' '0' ... 'ID' '0' 'ID']\n",
      " ['0' 'ID' '0' ... '0' '0' '0']\n",
      " ['0' '0' '0' ... '0' '0' '0']]\n",
      "11190\n",
      "11190\n"
     ]
    }
   ],
   "source": [
    "top_index_xb = np.argsort( -similarity_matrix, axis=1)\n",
    "words_dev_array = np.asarray(words_dev)\n",
    "words_entities_dev_array = np.asarray(words_entities_dev)\n",
    "words_dev_array[top_index_xb[:,:50]]\n",
    "ranked_entities = words_entities_dev_array[top_index_xb[:,:50]]\n",
    "print(ranked_entities)\n",
    "\n",
    "def most_frequent(List):\n",
    "    return max(set(List), key = List.count)\n",
    "ranked_entities = list(ranked_entities)\n",
    "most_frequent_entities = []\n",
    "for ranked_entities_one_token in ranked_entities:\n",
    "    most_frequent_entities.append( most_frequent( list(ranked_entities_one_token)) )\n",
    "print(len(words_input_sentence) )\n",
    "print(len(most_frequent_entities)) \n",
    "#print(most_frequent_entities)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "39b7f646",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kit: PART\n",
      "Install: OPER\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# get the output format like gpt \n",
    "i = 0\n",
    "response = []\n",
    "for sentence_index, sentence in enumerate(sentences_input):\n",
    "    response_one_sentence = \"\"\n",
    "    for token in sentence.split(\" \"):\n",
    "        if most_frequent_entities[i] == \"0\":\n",
    "            i = i + 1\n",
    "            continue\n",
    "        else:\n",
    "            response_one_sentence = response_one_sentence + token + \": \" + most_frequent_entities[i] + \"\\n\"\n",
    "        i = i + 1\n",
    "    response.append( response_one_sentence + \"\\n\")    \n",
    "print(response[0] ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "60ec2c86",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['TOOL', 'OPER', 'RPOS', 'ID', 'PART', 'PROP', 'WGT', 'DIM', 'QTY']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entity</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>F1 score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TOOL</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>72.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>OPER</td>\n",
       "      <td>62.795276</td>\n",
       "      <td>60.877863</td>\n",
       "      <td>0.618217</td>\n",
       "      <td>524.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>RPOS</td>\n",
       "      <td>67.706013</td>\n",
       "      <td>62.55144</td>\n",
       "      <td>0.650267</td>\n",
       "      <td>486.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID</td>\n",
       "      <td>100.0</td>\n",
       "      <td>1.538462</td>\n",
       "      <td>0.030303</td>\n",
       "      <td>65.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>PART</td>\n",
       "      <td>74.459273</td>\n",
       "      <td>78.164251</td>\n",
       "      <td>0.762668</td>\n",
       "      <td>2070.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>PROP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>55.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>WGT</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>DIM</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>35.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>QTY</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>61.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>accuracy</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>66.52819</td>\n",
       "      <td>3370.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>macro avg</td>\n",
       "      <td>33.884507</td>\n",
       "      <td>22.570224</td>\n",
       "      <td>0.229051</td>\n",
       "      <td>3370.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>weighted avg</td>\n",
       "      <td>67.193039</td>\n",
       "      <td>66.52819</td>\n",
       "      <td>0.658952</td>\n",
       "      <td>3370.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         entity  precision     recall  F1 score  support\n",
       "0          TOOL        0.0        0.0       0.0     72.0\n",
       "1          OPER  62.795276  60.877863  0.618217    524.0\n",
       "2          RPOS  67.706013   62.55144  0.650267    486.0\n",
       "3            ID      100.0   1.538462  0.030303     65.0\n",
       "4          PART  74.459273  78.164251  0.762668   2070.0\n",
       "5          PROP        0.0        0.0       0.0     55.0\n",
       "6           WGT        0.0        0.0       0.0      2.0\n",
       "7           DIM        0.0        0.0       0.0     35.0\n",
       "8           QTY        0.0        0.0       0.0     61.0\n",
       "0      accuracy          -          -  66.52819   3370.0\n",
       "1     macro avg  33.884507  22.570224  0.229051   3370.0\n",
       "2  weighted avg  67.193039   66.52819  0.658952   3370.0"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transformed_output = change_output_format_to_tokens_tags(sentences_input, response)\n",
    "transformed_solution = change_output_format_to_tokens_tags(sentences_input, demonstrations_input_solutions)\n",
    "get_evaluation_without_o(transformed_solution = transformed_solution, transformed_output = transformed_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7a49e3f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8dbc9237",
   "metadata": {},
   "outputs": [],
   "source": [
    "path_sen_train = \"data/immutable_data_formal/sen_assembly_train.txt\"\n",
    "path_dem_train = \"data/immutable_data_formal/dem_assembly_train.txt\"\n",
    "path_sen_input = \"data/immutable_data_formal/sen_assembly_input_30.txt\"\n",
    "path_dem_input = \"data/immutable_data_formal/dem_assembly_input_30.txt\"\n",
    "eva_baseline( path_sen_train, path_dem_train, path_sen_input, path_dem_input )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cce3c0ac",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04f75bda",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "5647faaa",
   "metadata": {},
   "source": [
    "# FabNER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "11da558c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array(['APPL', 'BIOP', 'CHAR', 'CONPRI', 'ENAT', 'FEAT', 'MACEQ', 'MANP',\n",
      "       'MANS', 'MATE', 'PARA', 'PRO'], dtype='<U1044'), array([ 2210,   236,  2226, 21113,  1238,  2610,  2551, 10622,    92,\n",
      "       12881,  3477,  5117], dtype=int64))\n",
      "Duplicated few-shot examples:  312\n",
      "length of stratified_sentences_develop:  309\n",
      "length of stratified_sentences_test:  500\n",
      "length of stratified_sentences_train:  13658\n"
     ]
    }
   ],
   "source": [
    "path_original_sentence = \"./data/fabNER/fabner_simple_total_original_sentence.jsonl\"\n",
    "path_entities = \"./data/fabNER/fabner_simple_total_entities.jsonl\"\n",
    "path_tokens_tags = \"./data/fabNER/fabner_simple_total.jsonl\"\n",
    "sentences_dev, sentences_input, sentences_train, demonstrations_dev, demonstrations_input_solutions, demonstrations_train = split_dev_dataset_input(path_original_sentence, path_entities, path_tokens_tags, num_input = 500, demon_size = 300, stratify = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fb0d88fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/immutable_data_formal/sen_fabner_input_30.txt\", \"rb\") as data:   #Pickling\n",
    "    sentences_input = pickle.load(data)  \n",
    "with open(\"data/immutable_data_formal/dem_fabner_input_30.txt\", \"rb\") as data:   #Pickling\n",
    "    demonstrations_input_solutions = pickle.load(data)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6e2db23b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9814 ['%', ')', 'was', 'mixed', 'in', 'AlSi10Mg', 'powder', 'by', 'conducting', 'low-energy']\n",
      "9814 ['0', '0', '0', '0', 'MANP', 'MATE', '0', '0', '0', '0']\n"
     ]
    }
   ],
   "source": [
    "words_dev = []\n",
    "words_entities_dev = []\n",
    "for i, sentence in enumerate(sentences_dev):\n",
    "    for token_index, token in enumerate(sentence.split(\" \")):\n",
    "        words_dev.append(token)\n",
    "        #get the list of entities for each sentence\n",
    "        for demon_row in demonstrations_dev[i].split(\"\\n\"):#### Long list of words and corresponding entities in development dataset\n",
    "            if token in demon_row.split(\": \")[0]:\n",
    "                words_entities_dev.append(demon_row.split(\": \")[1])\n",
    "                break\n",
    "        if len(words_dev) == len(words_entities_dev)+1:\n",
    "                words_entities_dev.append(\"0\")  \n",
    "print(len(words_dev), words_dev[0:10])\n",
    "print(len(words_entities_dev), words_entities_dev[0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a1950588",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14481\n"
     ]
    }
   ],
   "source": [
    "words_input_sentence = []\n",
    "for sentence in sentences_input:\n",
    "    #print(sentence)\n",
    "    for token in sentence.split(\" \"):\n",
    "        words_input_sentence.append(token)\n",
    "print(len(words_input_sentence))  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3227b0f7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████| 227/227 [01:29<00:00,  2.54it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████| 154/154 [01:04<00:00,  2.37it/s]\n"
     ]
    }
   ],
   "source": [
    "similarity_matrix = model.similarity(words_input_sentence, words_dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e6735c31",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['0' '0' '0' ... '0' '0' '0']\n",
      " ['0' 'FEAT' 'FEAT' ... '0' 'PARA' '0']\n",
      " ['0' '0' '0' ... 'CONPRI' '0' '0']\n",
      " ...\n",
      " ['0' '0' '0' ... 'CONPRI' '0' '0']\n",
      " ['0' '0' '0' ... '0' '0' '0']\n",
      " ['0' '0' '0' ... '0' '0' '0']]\n",
      "14481\n",
      "14481\n"
     ]
    }
   ],
   "source": [
    "top_index_xb = np.argsort( -similarity_matrix, axis=1)\n",
    "words_dev_array = np.asarray(words_dev)\n",
    "words_entities_dev_array = np.asarray(words_entities_dev)\n",
    "words_dev_array[top_index_xb[:,:50]]\n",
    "ranked_entities = words_entities_dev_array[top_index_xb[:,:50]]\n",
    "print(ranked_entities)\n",
    "\n",
    "def most_frequent(List):\n",
    "    return max(set(List), key = List.count)\n",
    "ranked_entities = list(ranked_entities)\n",
    "most_frequent_entities = []\n",
    "for ranked_entities_one_token in ranked_entities:\n",
    "    most_frequent_entities.append( most_frequent( list(ranked_entities_one_token)) )\n",
    "print(len(words_input_sentence) )\n",
    "print(len(most_frequent_entities)) \n",
    "#print(most_frequent_entities)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "92dfdd93",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a: CONPRI\n",
      "stages: CONPRI\n",
      "technology: CONPRI\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# get the output format like gpt \n",
    "i = 0\n",
    "response = []\n",
    "for sentence_index, sentence in enumerate(sentences_input):\n",
    "    response_one_sentence = \"\"\n",
    "    for token in sentence.split(\" \"):\n",
    "        if most_frequent_entities[i] == \"0\":\n",
    "            i = i + 1\n",
    "            continue\n",
    "        else:\n",
    "            response_one_sentence = response_one_sentence + token + \": \" + most_frequent_entities[i] + \"\\n\"\n",
    "        i = i + 1\n",
    "    response.append( response_one_sentence + \"\\n\")    \n",
    "print(response[0] ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "6a94c95d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['MANP', 'CHAR', 'ENAT', 'PRO', 'BIOP', 'CONPRI', 'MATE', 'MACEQ', 'PARA', 'APPL', 'MANS', 'FEAT']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entity</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>F1 score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MANP</td>\n",
       "      <td>47.928994</td>\n",
       "      <td>34.177215</td>\n",
       "      <td>0.399015</td>\n",
       "      <td>711.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CHAR</td>\n",
       "      <td>75.555556</td>\n",
       "      <td>17.435897</td>\n",
       "      <td>0.283333</td>\n",
       "      <td>195.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ENAT</td>\n",
       "      <td>12.5</td>\n",
       "      <td>1.923077</td>\n",
       "      <td>0.033333</td>\n",
       "      <td>52.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>PRO</td>\n",
       "      <td>42.990654</td>\n",
       "      <td>21.800948</td>\n",
       "      <td>0.289308</td>\n",
       "      <td>211.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>BIOP</td>\n",
       "      <td>11.764706</td>\n",
       "      <td>22.222222</td>\n",
       "      <td>0.153846</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>CONPRI</td>\n",
       "      <td>39.332366</td>\n",
       "      <td>28.952991</td>\n",
       "      <td>0.333538</td>\n",
       "      <td>936.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>MATE</td>\n",
       "      <td>56.774194</td>\n",
       "      <td>48.708487</td>\n",
       "      <td>0.52433</td>\n",
       "      <td>542.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>MACEQ</td>\n",
       "      <td>62.068966</td>\n",
       "      <td>14.173228</td>\n",
       "      <td>0.230769</td>\n",
       "      <td>127.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>PARA</td>\n",
       "      <td>26.229508</td>\n",
       "      <td>8.205128</td>\n",
       "      <td>0.125</td>\n",
       "      <td>195.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>APPL</td>\n",
       "      <td>15.686275</td>\n",
       "      <td>8.888889</td>\n",
       "      <td>0.113475</td>\n",
       "      <td>90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>MANS</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>FEAT</td>\n",
       "      <td>37.634409</td>\n",
       "      <td>22.875817</td>\n",
       "      <td>0.284553</td>\n",
       "      <td>153.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>accuracy</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>29.085271</td>\n",
       "      <td>3225.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>macro avg</td>\n",
       "      <td>35.705469</td>\n",
       "      <td>19.113658</td>\n",
       "      <td>0.230875</td>\n",
       "      <td>3225.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>weighted avg</td>\n",
       "      <td>45.392809</td>\n",
       "      <td>29.085271</td>\n",
       "      <td>0.343232</td>\n",
       "      <td>3225.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          entity  precision     recall   F1 score  support\n",
       "0           MANP  47.928994  34.177215   0.399015    711.0\n",
       "1           CHAR  75.555556  17.435897   0.283333    195.0\n",
       "2           ENAT       12.5   1.923077   0.033333     52.0\n",
       "3            PRO  42.990654  21.800948   0.289308    211.0\n",
       "4           BIOP  11.764706  22.222222   0.153846      9.0\n",
       "5         CONPRI  39.332366  28.952991   0.333538    936.0\n",
       "6           MATE  56.774194  48.708487    0.52433    542.0\n",
       "7          MACEQ  62.068966  14.173228   0.230769    127.0\n",
       "8           PARA  26.229508   8.205128      0.125    195.0\n",
       "9           APPL  15.686275   8.888889   0.113475     90.0\n",
       "10          MANS        0.0        0.0        0.0      4.0\n",
       "11          FEAT  37.634409  22.875817   0.284553    153.0\n",
       "0       accuracy          -          -  29.085271   3225.0\n",
       "1      macro avg  35.705469  19.113658   0.230875   3225.0\n",
       "2   weighted avg  45.392809  29.085271   0.343232   3225.0"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transformed_output = change_output_format_to_tokens_tags(sentences_input, response)\n",
    "transformed_solution = change_output_format_to_tokens_tags(sentences_input, demonstrations_input_solutions)\n",
    "get_evaluation_without_o(transformed_solution = transformed_solution, transformed_output = transformed_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6cd34660",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "051226d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "path_sen_train = \"data/immutable_data_formal/sen_fabner_train.txt\"\n",
    "path_dem_train = \"data/immutable_data_formal/dem_fabner_train.txt\"\n",
    "path_sen_input = \"data/immutable_data_formal/sen_fabner_input_30.txt\"\n",
    "path_dem_input = \"data/immutable_data_formal/dem_fabner_input_30.txt\"\n",
    "eva_baseline( path_sen_train, path_dem_train, path_sen_input, path_dem_input )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efc54aed",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".gptdata_hf38",
   "language": "python",
   "name": ".gptdata_hf38"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}