xlnet_prediction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Answer single Questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Didn't find file ./test-squad-trained_xlnet\\spiece.model. We won't load it.\n",
      "Didn't find file ./test-squad-trained_xlnet\\added_tokens.json. We won't load it.\n",
      "loading file None\n",
      "loading file ./test-squad-trained_xlnet\\tokenizer.json\n",
      "loading file None\n",
      "loading file ./test-squad-trained_xlnet\\special_tokens_map.json\n",
      "loading file ./test-squad-trained_xlnet\\tokenizer_config.json\n",
      "loading configuration file ./test-squad-trained_xlnet\\config.json\n",
      "Model config XLNetConfig {\n",
      "  \"_name_or_path\": \"./test-squad-trained_xlnet\",\n",
      "  \"architectures\": [\n",
      "    \"XLNetForQuestionAnsweringSimple\"\n",
      "  ],\n",
      "  \"attn_type\": \"bi\",\n",
      "  \"bi_data\": false,\n",
      "  \"bos_token_id\": 1,\n",
      "  \"clamp_len\": -1,\n",
      "  \"d_head\": 64,\n",
      "  \"d_inner\": 3072,\n",
      "  \"d_model\": 768,\n",
      "  \"dropout\": 0.1,\n",
      "  \"end_n_top\": 5,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"ff_activation\": \"gelu\",\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"mem_len\": null,\n",
      "  \"model_type\": \"xlnet\",\n",
      "  \"n_head\": 12,\n",
      "  \"n_layer\": 12,\n",
      "  \"pad_token_id\": 5,\n",
      "  \"reuse_len\": null,\n",
      "  \"same_length\": false,\n",
      "  \"start_n_top\": 5,\n",
      "  \"summary_activation\": \"tanh\",\n",
      "  \"summary_last_dropout\": 0.1,\n",
      "  \"summary_type\": \"last\",\n",
      "  \"summary_use_proj\": true,\n",
      "  \"task_specific_params\": {\n",
      "    \"text-generation\": {\n",
      "      \"do_sample\": true,\n",
      "      \"max_length\": 250\n",
      "    }\n",
      "  },\n",
      "  \"torch_dtype\": \"float32\",\n",
      "  \"transformers_version\": \"4.15.0\",\n",
      "  \"untie_r\": true,\n",
      "  \"use_mems_eval\": true,\n",
      "  \"use_mems_train\": false,\n",
      "  \"vocab_size\": 32000\n",
      "}\n",
      "\n",
      "loading weights file ./test-squad-trained_xlnet\\pytorch_model.bin\n",
      "All model checkpoint weights were used when initializing XLNetForQuestionAnsweringSimple.\n",
      "\n",
      "All the weights of XLNetForQuestionAnsweringSimple were initialized from the model checkpoint at ./test-squad-trained_xlnet.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use XLNetForQuestionAnsweringSimple for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "import xlnet\n",
    "from utils import *\n",
    "\n",
    "testmodel = xlnet.XLNET()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'score': 0.9578208327293396, 'start': 0, 'end': 0, 'answer': ''}"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testmodel.predict(\"who am I?\", \"My name is Roman.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'score': 1.0, 'start': 0, 'end': 0, 'answer': ''}"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testmodel.predict(\"who am I?\", \"My name is Roman but not Vincenzo.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'score': 0.9999890923500061, 'start': 0, 'end': 0, 'answer': ''}"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testmodel.predict(\"who am I?\", \"My name is Roman and Vincenzo.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'score': 1.0, 'start': 0, 'end': 0, 'answer': ''}"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testmodel.predict(\"who am I?\", \"My name is not Roman and but Vincenzo.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Answer multiple Questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from transformers import Trainer, default_data_collator, AutoModelForQuestionAnswering, AutoTokenizer, TrainingArguments,XLNetTokenizer\n",
    "from datasets import load_dataset, load_metric\n",
    "import torch\n",
    "import numpy as np\n",
    "from utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reusing dataset squad_v2 (C:\\Users\\vince\\.cache\\huggingface\\datasets\\squad_v2\\squad_v2\\2.0.0\\09187c73c1b837c95d9a249cd97c2c3f1cebada06efe667b4427714b27639b1d)\n",
      "100%|██████████| 2/2 [00:00<00:00, 117.65it/s]\n"
     ]
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"xlnet-base-cased\")\n",
    "model = AutoModelForQuestionAnswering.from_pretrained(\"./test-squad-trained_xlnet\").to(\"cuda:0\")\n",
    "\n",
    "config = load_yaml_file('xlnet_config.yaml')\n",
    "\n",
    "##\n",
    "GPU_USAGE = config['GPU_USAGE']\n",
    "BATCH_SIZE = config['BATCH_SIZE']\n",
    "MAX_LENGTH = config['MAX_LENGTH']\n",
    "DOC_STRIDE = config['DOC_STRIDE']\n",
    "N_BEST_SIZE = config['N_BEST_SIZE']\n",
    "MAX_ANSWER_LENGTH = config['MAX_ANSWER_LENGTH']\n",
    "squad_v2 = config['squad_v2']\n",
    "##\n",
    "\n",
    "PAD_RIGHT = tokenizer.padding_side == \"right\"\n",
    "\n",
    "datasets = load_dataset(\"squad_v2\")\n",
    "data_collator = default_data_collator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at C:\\Users\\vince\\.cache\\huggingface\\datasets\\squad_v2\\squad_v2\\2.0.0\\09187c73c1b837c95d9a249cd97c2c3f1cebada06efe667b4427714b27639b1d\\cache-f7f821988ca865d4.arrow\n",
      "Loading cached processed dataset at C:\\Users\\vince\\.cache\\huggingface\\datasets\\squad_v2\\squad_v2\\2.0.0\\09187c73c1b837c95d9a249cd97c2c3f1cebada06efe667b4427714b27639b1d\\cache-c9925728a42415f7.arrow\n"
     ]
    }
   ],
   "source": [
    "tokenized_datasets = datasets.map(\n",
    "    prepare_train_features, \n",
    "    fn_kwargs={\n",
    "        'tokenizer':tokenizer, \n",
    "        'PAD_RIGHT':PAD_RIGHT,\n",
    "        'MAX_LENGTH':MAX_LENGTH, \n",
    "        'DOC_STRIDE':DOC_STRIDE\n",
    "        }, \n",
    "    batched=True, remove_columns=datasets[\"train\"].column_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using the `WAND_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n"
     ]
    }
   ],
   "source": [
    "model_name = \"test-squad-trained\"\n",
    "\n",
    "import os\n",
    "os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
    "\n",
    "args = TrainingArguments(\n",
    "    f\"{model_name}-finetuned-squad\",\n",
    "    evaluation_strategy = \"epoch\",\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=BATCH_SIZE,\n",
    "    per_device_eval_batch_size=BATCH_SIZE,\n",
    "    num_train_epochs=3,\n",
    "    weight_decay=0.01,\n",
    "    push_to_hub=False,\n",
    "\n",
    ")\n",
    "\n",
    "data_collator = default_data_collator\n",
    "\n",
    "trainer = Trainer(\n",
    "    model,\n",
    "    args,\n",
    "    train_dataset=tokenized_datasets[\"train\"],\n",
    "    eval_dataset=tokenized_datasets[\"validation\"],\n",
    "    data_collator=data_collator,\n",
    "    tokenizer=tokenizer,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at C:\\Users\\vince\\.cache\\huggingface\\datasets\\squad_v2\\squad_v2\\2.0.0\\09187c73c1b837c95d9a249cd97c2c3f1cebada06efe667b4427714b27639b1d\\cache-b8ab0739473eb6a6.arrow\n"
     ]
    }
   ],
   "source": [
    "validation_features = datasets[\"validation\"].map(\n",
    "    prepare_validation_features,\n",
    "    fn_kwargs={\n",
    "        'tokenizer':tokenizer, \n",
    "        'PAD_RIGHT':PAD_RIGHT, \n",
    "        'MAX_LENGTH':MAX_LENGTH, \n",
    "        'DOC_STRIDE':DOC_STRIDE\n",
    "        },\n",
    "    batched=True,\n",
    "    remove_columns=datasets[\"validation\"].column_names\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the test set  don't have a corresponding argument in `XLNetForQuestionAnsweringSimple.forward` and have been ignored: example_id, offset_mapping.\n",
      "***** Running Prediction *****\n",
      "  Num examples = 12852\n",
      "  Batch size = 8\n",
      "100%|█████████▉| 1606/1607 [04:58<00:00,  7.08it/s]"
     ]
    }
   ],
   "source": [
    "raw_predictions = trainer.predict(validation_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "odict_keys(['loss', 'start_logits', 'end_logits', 'mems'])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for batch in trainer.get_eval_dataloader():\n",
    "    break\n",
    "batch = {k: v.to(trainer.args.device) for k, v in batch.items()}\n",
    "with torch.no_grad():\n",
    "    output = trainer.model(**batch)\n",
    "output.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'score': 4.275517, 'text': 'France'},\n",
       " {'score': -3.3379405, 'text': 'France.'},\n",
       " {'score': -4.914199, 'text': 'a region in France'},\n",
       " {'score': -5.3332705, 'text': 'region in France'},\n",
       " {'score': -5.577037, 'text': 'Normandy, a region in France'},\n",
       " {'score': -6.4357605, 'text': 'in France'},\n",
       " {'score': -7.476758, 'text': ', a region in France'},\n",
       " {'score': -8.758042,\n",
       "  'text': '10th and 11th centuries gave their name to Normandy, a region in France'},\n",
       " {'score': -9.317292,\n",
       "  'text': 'the 10th and 11th centuries gave their name to Normandy, a region in France'},\n",
       " {'score': -9.40016,\n",
       "  'text': 'in the 10th and 11th centuries gave their name to Normandy, a region in France'},\n",
       " {'score': -9.456315,\n",
       "  'text': 'Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France'},\n",
       " {'score': -10.007275,\n",
       "  'text': 'gave their name to Normandy, a region in France'},\n",
       " {'score': -10.153693,\n",
       "  'text': 'Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France'},\n",
       " {'score': -12.527657, 'text': 'a region in France.'},\n",
       " {'score': -12.946728, 'text': 'region in France.'},\n",
       " {'score': -13.190495, 'text': 'Normandy, a region in France.'},\n",
       " {'score': -13.8294735, 'text': '.'},\n",
       " {'score': -14.049218, 'text': 'in France.'},\n",
       " {'score': -14.650492, 'text': 'Normandy'},\n",
       " {'score': -15.090216, 'text': ', a region in France.'}]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "start_logits = output.start_logits[0].cpu().numpy()\n",
    "end_logits = output.end_logits[0].cpu().numpy()\n",
    "offset_mapping = validation_features[0][\"offset_mapping\"]\n",
    "# The first feature comes from the first example. For the more general case, we will need to be match the example_id to\n",
    "# an example index\n",
    "context = datasets[\"validation\"][0][\"context\"]\n",
    "\n",
    "# Gather the indices the best start/end logits:\n",
    "start_indexes = np.argsort(start_logits)[-1 : -N_BEST_SIZE - 1 : -1].tolist()\n",
    "end_indexes = np.argsort(end_logits)[-1 : -N_BEST_SIZE - 1 : -1].tolist()\n",
    "valid_answers = []\n",
    "for start_index in start_indexes:\n",
    "    for end_index in end_indexes:\n",
    "        # Don't consider out-of-scope answers, either because the indices are out of bounds or correspond\n",
    "        # to part of the input_ids that are not in the context.\n",
    "        if (\n",
    "                start_index >= len(offset_mapping)\n",
    "                or end_index >= len(offset_mapping)\n",
    "                or offset_mapping[start_index] is None\n",
    "                or offset_mapping[end_index] is None\n",
    "        ):\n",
    "            continue\n",
    "        # Don't consider answers with a length that is either < 0 or > max_answer_length.\n",
    "        if end_index < start_index or end_index - start_index + 1 > MAX_ANSWER_LENGTH:\n",
    "            continue\n",
    "        if start_index <= end_index: # We need to refine that test to check the answer is inside the context\n",
    "            start_char = offset_mapping[start_index][0]\n",
    "            end_char = offset_mapping[end_index][1]\n",
    "            valid_answers.append(\n",
    "                {\n",
    "                    \"score\": start_logits[start_index] + end_logits[end_index],\n",
    "                    \"text\": context[start_char: end_char]\n",
    "                }\n",
    "            )\n",
    "\n",
    "valid_answers = sorted(valid_answers, key=lambda x: x[\"score\"], reverse=True)[:N_BEST_SIZE  ]\n",
    "valid_answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Post-processing 11873 example predictions split into 12852 features.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 11873/11873 [00:23<00:00, 501.37it/s]\n"
     ]
    }
   ],
   "source": [
    "final_predictions = postprocess_qa_predictions(\n",
    "    datasets[\"validation\"], \n",
    "    validation_features, \n",
    "    raw_predictions.predictions,\n",
    "    tokenizer, \n",
    "    config['squad_v2'], \n",
    "    n_best_size=config['N_BEST_SIZE'], \n",
    "    max_answer_length=config['MAX_ANSWER_LENGTH'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'exact': 55.50408489850922,\n",
       " 'f1': 56.9753646445723,\n",
       " 'total': 11873,\n",
       " 'HasAns_exact': 17.07152496626181,\n",
       " 'HasAns_f1': 20.018303715419414,\n",
       " 'HasAns_total': 5928,\n",
       " 'NoAns_exact': 93.82674516400337,\n",
       " 'NoAns_f1': 93.82674516400337,\n",
       " 'NoAns_total': 5945,\n",
       " 'best_exact': 55.50408489850922,\n",
       " 'best_exact_thresh': 0.0,\n",
       " 'best_f1': 56.975364644572494,\n",
       " 'best_f1_thresh': 0.0}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metric = load_metric(\"squad_v2\" if squad_v2 else \"squad\")\n",
    "if squad_v2:\n",
    "    formatted_predictions = [{\"id\": k, \"prediction_text\": v, \"no_answer_probability\": 0.0} for k, v in final_predictions.items()]\n",
    "else:\n",
    "    formatted_predictions = [{\"id\": k, \"prediction_text\": v} for k, v in final_predictions.items()]\n",
    "references = [{\"id\": ex[\"id\"], \"answers\": ex[\"answers\"]} for ex in datasets[\"validation\"]]\n",
    "metric.compute(predictions=formatted_predictions, references=references)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BLEU-2 avg: 0.8066689986769077 of 1424 comparisons\n"
     ]
    }
   ],
   "source": [
    "bleus = calculate_bleu_score(formatted_predictions, references)\n",
    "bleu_avg = np.sum(bleus)/len(bleus)\n",
    "print('BLEU-2 avg:', bleu_avg, 'of', len(bleus), 'comparisons')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BLEU-2 avg: 0.566554085245171 of 11873 comparisons\n"
     ]
    }
   ],
   "source": [
    "bleus = calculate_bleu_score_new(formatted_predictions, references)\n",
    "bleu_avg = np.sum(bleus)/len(bleus)\n",
    "print('BLEU-2 avg:', bleu_avg, 'of', len(bleus), 'comparisons')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1424 tp | 367 fp\n",
      "-----------------\n",
      "4504 fn | 5578 tn\n"
     ]
    }
   ],
   "source": [
    "tp, fp, tn, fn = [], [], [], []\n",
    "for i in range(len(formatted_predictions)):\n",
    "    pred = formatted_predictions[i]['prediction_text']\n",
    "    ref = references[i]['answers']['text']\n",
    "    if (len(pred) == 0) and (len(ref) == 0):\n",
    "        tn.append(i)\n",
    "    elif (len(pred) != 0) and (len(ref) == 0):\n",
    "        fp.append(i)\n",
    "    elif (len(pred) == 0) and (len(ref) != 0):\n",
    "        fn.append(i)\n",
    "    elif (len(pred) != 0) and (len(ref) != 0):\n",
    "        tp.append(i)\n",
    "    \n",
    "print('{} tp | {} fp\\n-----------------\\n{} fn | {} tn'.format(len(tp),len(fp),len(fn),len(tn)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5873\n",
      "{'id': '5a25bd5cef59cd001a623cc6', 'title': 'Construction', 'context': \"New techniques of building construction are being researched, made possible by advances in 3D printing technology. In a form of additive building construction, similar to the additive manufacturing techniques for manufactured parts, building printing is making it possible to flexibly construct small commercial buildings and private habitations in around 20 hours, with built-in plumbing and electrical facilities, in one continuous build, using large 3D printers. Working versions of 3D-printing building technology are already printing 2 metres (6 ft 7 in) of building material per hour as of January 2013[update], with the next-generation printers capable of 3.5 metres (11 ft) per hour, sufficient to complete a building in a week. Dutch architect Janjaap Ruijssenaars's performative architecture 3D-printed building is scheduled to be built in 2014.\", 'question': 'What is flexible construction similar to?', 'answers': {'text': [], 'answer_start': []}}\n",
      "\n",
      "{'id': '5a25bd5cef59cd001a623cc6', 'prediction_text': 'additive manufacturing techniques for manufactured parts', 'no_answer_probability': 0.0}\n",
      "\n",
      "{'id': '5a25bd5cef59cd001a623cc6', 'answers': {'text': [], 'answer_start': []}}\n",
      "\n",
      "\n",
      "\n",
      "1474\n",
      "{'id': '5ad263d9d7d075001a429124', 'title': 'Huguenot', 'context': 'In the early years, many Huguenots also settled in the area of present-day Charleston, South Carolina. In 1685, Rev. Elie Prioleau from the town of Pons in France, was among the first to settle there. He became pastor of the first Huguenot church in North America in that city. After the Revocation of the Edict of Nantes in 1685, several Huguenot families of Norman and Carolingian nobility and descent, including Edmund Bohun of Suffolk England from the Humphrey de Bohun line of French royalty descended from Charlemagne, Jean Postell of Dieppe France, Alexander Pepin, Antoine Poitevin of Orsement France, and Jacques de Bordeaux of Grenoble, immigrated to the Charleston Orange district. They were very successful at marriage and property speculation. After petitioning the British Crown in 1697 for the right to own land in the Baronies, they prospered as slave owners on the Cooper, Ashepoo, Ashley and Santee River plantations they purchased from the British Landgrave Edmund Bellinger. Some of their descendants moved into the Deep South and Texas, where they developed new plantations.', 'question': 'In what year did Rev. Elie Prioleau become the pastor of the first Huguenot church in Charleston, South Carolina?', 'answers': {'text': [], 'answer_start': []}}\n",
      "\n",
      "{'id': '5ad263d9d7d075001a429124', 'prediction_text': '1685', 'no_answer_probability': 0.0}\n",
      "\n",
      "{'id': '5ad263d9d7d075001a429124', 'answers': {'text': [], 'answer_start': []}}\n",
      "\n",
      "\n",
      "\n",
      "690\n",
      "{'id': '5ad0220a77cf76001a686b2b', 'title': 'Southern_California', 'context': \"Southern California is also home to a large home grown surf and skateboard culture. Companies such as Volcom, Quiksilver, No Fear, RVCA, and Body Glove are all headquartered here. Professional skateboarder Tony Hawk, professional surfers Rob Machado, Tim Curran, Bobby Martinez, Pat O'Connell, Dane Reynolds, and Chris Ward, and professional snowboarder Shaun White live in southern California. Some of the world's legendary surf spots are in southern California as well, including Trestles, Rincon, The Wedge, Huntington Beach, and Malibu, and it is second only to the island of Oahu in terms of famous surf breaks. Some of the world's biggest extreme sports events, including the X Games, Boost Mobile Pro, and the U.S. Open of Surfing are all in southern California. Southern California is also important to the world of yachting. The annual Transpacific Yacht Race, or Transpac, from Los Angeles to Hawaii, is one of yachting's premier events. The San Diego Yacht Club held the America's Cup, the most prestigious prize in yachting, from 1988 to 1995 and hosted three America's Cup races during that time.\", 'question': 'Who are Rob Curran and Tim Machado?', 'answers': {'text': [], 'answer_start': []}}\n",
      "\n",
      "{'id': '5ad0220a77cf76001a686b2b', 'prediction_text': 'professional surfers', 'no_answer_probability': 0.0}\n",
      "\n",
      "{'id': '5ad0220a77cf76001a686b2b', 'answers': {'text': [], 'answer_start': []}}\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "fp_n = 3\n",
    "np.random.seed(42)\n",
    "idxs_fp = np.random.choice(len(fp),fp_n,False)\n",
    "\n",
    "for idx_fp in idxs_fp:\n",
    "    idx_fp = fp[int(idx_fp)]\n",
    "    print(str(idx_fp))\n",
    "    print(str(datasets['validation'][idx_fp]))\n",
    "    print('\\n'+str(formatted_predictions[idx_fp]))\n",
    "    print('\\n'+str(references[idx_fp])+'\\n\\n\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5925\n",
      "{'id': '572750e8dd62a815002e9af4', 'title': 'Construction', 'context': 'The project must adhere to zoning and building code requirements. Constructing a project that fails to adhere to codes does not benefit the owner. Some legal requirements come from malum in se considerations, or the desire to prevent things that are indisputably bad – bridge collapses or explosions. Other legal requirements come from malum prohibitum considerations, or things that are a matter of custom or expectation, such as isolating businesses to a business district and residences to a residential district. An attorney may seek changes or exemptions in the law that governs the land where the building will be built, either by arguing that a rule is inapplicable (the bridge design will not cause a collapse), or that the custom is no longer needed (acceptance of live-work spaces has grown in the community).', 'question': 'Who may seek changes or exemptions in the law that governs the land where the building will be built?', 'answers': {'text': ['An attorney', 'attorney', 'An attorney'], 'answer_start': [517, 520, 517]}}\n",
      "\n",
      "{'id': '572750e8dd62a815002e9af4', 'prediction_text': '', 'no_answer_probability': 0.0}\n",
      "\n",
      "{'id': '572750e8dd62a815002e9af4', 'answers': {'text': ['An attorney', 'attorney', 'An attorney'], 'answer_start': [517, 520, 517]}}\n",
      "\n",
      "\n",
      "\n",
      "7798\n",
      "{'id': '572879574b864d1900164a17', 'title': 'Yuan_dynasty', 'context': 'Western musical instruments were introduced to enrich Chinese performing arts. From this period dates the conversion to Islam, by Muslims of Central Asia, of growing numbers of Chinese in the northwest and southwest. Nestorianism and Roman Catholicism also enjoyed a period of toleration. Buddhism (especially Tibetan Buddhism) flourished, although Taoism endured certain persecutions in favor of Buddhism from the Yuan government. Confucian governmental practices and examinations based on the Classics, which had fallen into disuse in north China during the period of disunity, were reinstated by the Yuan court, probably in the hope of maintaining order over Han society. Advances were realized in the fields of travel literature, cartography, geography, and scientific education.', 'question': 'What type of practices did the Yuan reintroduce in government?', 'answers': {'text': ['Confucian', 'Confucian governmental practices and examinations', 'Confucian'], 'answer_start': [432, 432, 432]}}\n",
      "\n",
      "{'id': '572879574b864d1900164a17', 'prediction_text': '', 'no_answer_probability': 0.0}\n",
      "\n",
      "{'id': '572879574b864d1900164a17', 'answers': {'text': ['Confucian', 'Confucian governmental practices and examinations', 'Confucian'], 'answer_start': [432, 432, 432]}}\n",
      "\n",
      "\n",
      "\n",
      "7198\n",
      "{'id': '572a213e6aef0514001552f0', 'title': 'Economic_inequality', 'context': 'The capabilities approach – sometimes called the human development approach – looks at income inequality and poverty as form of “capability deprivation”. Unlike neoliberalism, which “defines well-being as utility maximization”, economic growth and income are considered a means to an end rather than the end itself. Its goal is to “wid[en] people’s choices and the level of their achieved well-being” through increasing functionings (the things a person values doing), capabilities (the freedom to enjoy functionings) and agency (the ability to pursue valued goals).', 'question': 'What is the goal of the capabilities approach?', 'answers': {'text': ['to “wid[en] people’s choices and the level of their achieved well-being”', 'wid[en] people’s choices and the level of their achieved well-being', '“wid[en] people’s choices and the level of their achieved well-being”'], 'answer_start': [328, 332, 331]}}\n",
      "\n",
      "{'id': '572a213e6aef0514001552f0', 'prediction_text': '', 'no_answer_probability': 0.0}\n",
      "\n",
      "{'id': '572a213e6aef0514001552f0', 'answers': {'text': ['to “wid[en] people’s choices and the level of their achieved well-being”', 'wid[en] people’s choices and the level of their achieved well-being', '“wid[en] people’s choices and the level of their achieved well-being”'], 'answer_start': [328, 332, 331]}}\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "fn_n = 3\n",
    "np.random.seed(42)\n",
    "idxs_fn = np.random.choice(len(fn),fn_n,False)\n",
    "\n",
    "for idx_fn in idxs_fn:\n",
    "    idx_fn = fn[int(idx_fn)]\n",
    "    print(str(idx_fn))\n",
    "    print(str(datasets['validation'][idx_fn]))\n",
    "    print('\\n'+str(formatted_predictions[idx_fn]))\n",
    "    print('\\n'+str(references[idx_fn])+'\\n\\n\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading configuration file test-squad-trained_xlnet\\config.json\n",
      "Model config XLNetConfig {\n",
      "  \"_name_or_path\": \"test-squad-trained_xlnet\",\n",
      "  \"architectures\": [\n",
      "    \"XLNetForQuestionAnsweringSimple\"\n",
      "  ],\n",
      "  \"attn_type\": \"bi\",\n",
      "  \"bi_data\": false,\n",
      "  \"bos_token_id\": 1,\n",
      "  \"clamp_len\": -1,\n",
      "  \"d_head\": 64,\n",
      "  \"d_inner\": 3072,\n",
      "  \"d_model\": 768,\n",
      "  \"dropout\": 0.1,\n",
      "  \"end_n_top\": 5,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"ff_activation\": \"gelu\",\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"mem_len\": null,\n",
      "  \"model_type\": \"xlnet\",\n",
      "  \"n_head\": 12,\n",
      "  \"n_layer\": 12,\n",
      "  \"pad_token_id\": 5,\n",
      "  \"reuse_len\": null,\n",
      "  \"same_length\": false,\n",
      "  \"start_n_top\": 5,\n",
      "  \"summary_activation\": \"tanh\",\n",
      "  \"summary_last_dropout\": 0.1,\n",
      "  \"summary_type\": \"last\",\n",
      "  \"summary_use_proj\": true,\n",
      "  \"task_specific_params\": {\n",
      "    \"text-generation\": {\n",
      "      \"do_sample\": true,\n",
      "      \"max_length\": 250\n",
      "    }\n",
      "  },\n",
      "  \"torch_dtype\": \"float32\",\n",
      "  \"transformers_version\": \"4.15.0\",\n",
      "  \"untie_r\": true,\n",
      "  \"use_mems_eval\": true,\n",
      "  \"use_mems_train\": false,\n",
      "  \"vocab_size\": 32000\n",
      "}\n",
      "\n",
      "loading weights file test-squad-trained_xlnet\\pytorch_model.bin\n",
      "All model checkpoint weights were used when initializing XLNetForQuestionAnsweringSimple.\n",
      "\n",
      "All the weights of XLNetForQuestionAnsweringSimple were initialized from the model checkpoint at test-squad-trained_xlnet.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use XLNetForQuestionAnsweringSimple for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "model_cpu = AutoModelForQuestionAnswering.from_pretrained(\"test-squad-trained_xlnet\")\n",
    "qa_prediction = pipeline('question-answering', model=model_cpu, tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}