notebooks/Identifying model configuration.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b761d256",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Some weights of the model checkpoint at microsoft/deberta-v3-small were not used when initializing DebertaV2Model: ['mask_predictions.classifier.weight', 'lm_predictions.lm_head.LayerNorm.bias', 'mask_predictions.LayerNorm.weight', 'mask_predictions.dense.bias', 'lm_predictions.lm_head.dense.bias', 'mask_predictions.dense.weight', 'lm_predictions.lm_head.dense.weight', 'lm_predictions.lm_head.bias', 'mask_predictions.classifier.bias', 'lm_predictions.lm_head.LayerNorm.weight', 'mask_predictions.LayerNorm.bias']\n",
      "- This IS expected if you are initializing DebertaV2Model from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DebertaV2Model from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "import ecco\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM, AutoModel, AutoModelForSeq2SeqLM\n",
    "hf_model_id = 'microsoft/deberta-v3-small'\n",
    "\n",
    "\n",
    "# 1- load the model the tokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(hf_model_id)\n",
    "model = AutoModel.from_pretrained(hf_model_id)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dde337e4",
   "metadata": {},
   "source": [
    "## 2- identify model type\n",
    "Identify type of model. Is it `causal` like GPT? `mlm` like BERT, or `enc-dec` like T5? We learn this by reading about the model artchitecture. In this example, deBERTa is based on BERT and so is `mlm`.\n",
    "\n",
    "## 3- identify layer names\n",
    "We need the names of the embedding layer (to support attribution calculations) and the FFNN layer (to support neuron activation collection)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bf98b88d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DebertaV2Model(\n",
       "  (embeddings): DebertaV2Embeddings(\n",
       "    (word_embeddings): Embedding(128100, 768, padding_idx=0)\n",
       "    (LayerNorm): LayerNorm((768,), eps=1e-07, elementwise_affine=True)\n",
       "    (dropout): StableDropout()\n",
       "  )\n",
       "  (encoder): DebertaV2Encoder(\n",
       "    (layer): ModuleList(\n",
       "      (0): DebertaV2Layer(\n",
       "        (attention): DebertaV2Attention(\n",
       "          (self): DisentangledSelfAttention(\n",
       "            (query_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (pos_dropout): StableDropout()\n",
       "            (dropout): StableDropout()\n",
       "          )\n",
       "          (output): DebertaV2SelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-07, elementwise_affine=True)\n",
       "            (dropout): StableDropout()\n",
       "          )\n",
       "        )\n",
       "        (intermediate): DebertaV2Intermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): DebertaV2Output(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): LayerNorm((768,), eps=1e-07, elementwise_affine=True)\n",
       "          (dropout): StableDropout()\n",
       "        )\n",
       "      )\n",
       "      (1): DebertaV2Layer(\n",
       "        (attention): DebertaV2Attention(\n",
       "          (self): DisentangledSelfAttention(\n",
       "            (query_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (pos_dropout): StableDropout()\n",
       "            (dropout): StableDropout()\n",
       "          )\n",
       "          (output): DebertaV2SelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-07, elementwise_affine=True)\n",
       "            (dropout): StableDropout()\n",
       "          )\n",
       "        )\n",
       "        (intermediate): DebertaV2Intermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): DebertaV2Output(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): LayerNorm((768,), eps=1e-07, elementwise_affine=True)\n",
       "          (dropout): StableDropout()\n",
       "        )\n",
       "      )\n",
       "      (2): DebertaV2Layer(\n",
       "        (attention): DebertaV2Attention(\n",
       "          (self): DisentangledSelfAttention(\n",
       "            (query_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (pos_dropout): StableDropout()\n",
       "            (dropout): StableDropout()\n",
       "          )\n",
       "          (output): DebertaV2SelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-07, elementwise_affine=True)\n",
       "            (dropout): StableDropout()\n",
       "          )\n",
       "        )\n",
       "        (intermediate): DebertaV2Intermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): DebertaV2Output(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): LayerNorm((768,), eps=1e-07, elementwise_affine=True)\n",
       "          (dropout): StableDropout()\n",
       "        )\n",
       "      )\n",
       "      (3): DebertaV2Layer(\n",
       "        (attention): DebertaV2Attention(\n",
       "          (self): DisentangledSelfAttention(\n",
       "            (query_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (pos_dropout): StableDropout()\n",
       "            (dropout): StableDropout()\n",
       "          )\n",
       "          (output): DebertaV2SelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-07, elementwise_affine=True)\n",
       "            (dropout): StableDropout()\n",
       "          )\n",
       "        )\n",
       "        (intermediate): DebertaV2Intermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): DebertaV2Output(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): LayerNorm((768,), eps=1e-07, elementwise_affine=True)\n",
       "          (dropout): StableDropout()\n",
       "        )\n",
       "      )\n",
       "      (4): DebertaV2Layer(\n",
       "        (attention): DebertaV2Attention(\n",
       "          (self): DisentangledSelfAttention(\n",
       "            (query_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (pos_dropout): StableDropout()\n",
       "            (dropout): StableDropout()\n",
       "          )\n",
       "          (output): DebertaV2SelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-07, elementwise_affine=True)\n",
       "            (dropout): StableDropout()\n",
       "          )\n",
       "        )\n",
       "        (intermediate): DebertaV2Intermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): DebertaV2Output(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): LayerNorm((768,), eps=1e-07, elementwise_affine=True)\n",
       "          (dropout): StableDropout()\n",
       "        )\n",
       "      )\n",
       "      (5): DebertaV2Layer(\n",
       "        (attention): DebertaV2Attention(\n",
       "          (self): DisentangledSelfAttention(\n",
       "            (query_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (pos_dropout): StableDropout()\n",
       "            (dropout): StableDropout()\n",
       "          )\n",
       "          (output): DebertaV2SelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-07, elementwise_affine=True)\n",
       "            (dropout): StableDropout()\n",
       "          )\n",
       "        )\n",
       "        (intermediate): DebertaV2Intermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): DebertaV2Output(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): LayerNorm((768,), eps=1e-07, elementwise_affine=True)\n",
       "          (dropout): StableDropout()\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (rel_embeddings): Embedding(512, 768)\n",
       "    (LayerNorm): LayerNorm((768,), eps=1e-07, elementwise_affine=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f91fc844",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Embedding(128100, 768, padding_idx=0)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Embeddings is:\n",
    "model.embeddings.word_embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "279dd1b7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Linear(in_features=768, out_features=3072, bias=True)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The FFNN is the one with the widest output dimension\n",
    "# We take the parts of the name after the layer number. In this case it's \n",
    "# intermediate.dense\n",
    "model.encoder.layer[0].intermediate.dense"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f168e628",
   "metadata": {},
   "source": [
    "## Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "5662cac2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': [1, 10704, 4820, 2], 'token_type_ids': [0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1]}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ids= tokenizer('tokenization')\n",
    "ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "8eaef886",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[CLS]', '▁token', 'ization', '[SEP]']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.convert_ids_to_tokens(ids['input_ids'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a2b999b",
   "metadata": {},
   "source": [
    "So `token_prefix` is ▁\n",
    "\n",
    "And `partial_token_prefix` is ''"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93b6f14c",
   "metadata": {},
   "source": [
    "# Model config\n",
    "If we're passing the model config directly to Ecco, it would look like this:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9d1df40b",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "model_config = {\n",
    "    'embedding': \"embeddings.word_embeddings\",\n",
    "    'type': 'mlm',\n",
    "    'activations': ['intermediate\\.dense'], #This is a regex\n",
    "    'token_prefix': '▁',\n",
    "    'partial_token_prefix': ''\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9d98ca10",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Some weights of the model checkpoint at microsoft/deberta-v3-small were not used when initializing DebertaV2Model: ['mask_predictions.classifier.weight', 'lm_predictions.lm_head.LayerNorm.bias', 'mask_predictions.LayerNorm.weight', 'mask_predictions.dense.bias', 'lm_predictions.lm_head.dense.bias', 'mask_predictions.dense.weight', 'lm_predictions.lm_head.dense.weight', 'lm_predictions.lm_head.bias', 'mask_predictions.classifier.bias', 'lm_predictions.lm_head.LayerNorm.weight', 'mask_predictions.LayerNorm.bias']\n",
      "- This IS expected if you are initializing DebertaV2Model from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DebertaV2Model from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "ecco_model = ecco.from_pretrained(hf_model_id, \n",
    "                                  activations=True,\n",
    "                                  model_config=model_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "45925d81",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'output' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-4-3d050f37da8c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0moutput\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'output' is not defined"
     ]
    }
   ],
   "source": [
    "output.config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0befcba1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<html lang=\"en\">\n",
       "<script src=\"https://requirejs.org/docs/release/2.3.6/minified/require.js\"></script>\n",
       "<script>\n",
       "    //var ecco_url = 'https://storage.googleapis.com/ml-intro/ecco/'\n",
       "    var ecco_url = 'http://localhost:8000/'\n",
       "\n",
       "    if (window.ecco === undefined) window.ecco = {}\n",
       "\n",
       "    // Setup the paths of the script we'll be using\n",
       "    requirejs.config({\n",
       "        urlArgs: \"bust=\" + (new Date()).getTime(),\n",
       "        nodeRequire: require,\n",
       "        paths: {\n",
       "            d3: \"https://d3js.org/d3.v6.min\", // This is only for use in setup.html and basic.html\n",
       "            \"d3-array\": \"https://d3js.org/d3-array.v2.min\",\n",
       "            jquery: \"https://code.jquery.com/jquery-3.5.1.min\",\n",
       "            ecco: ecco_url + 'js/0.0.5/ecco-bundle.min',\n",
       "            xregexp: 'https://cdnjs.cloudflare.com/ajax/libs/xregexp/3.2.0/xregexp-all.min'\n",
       "        }\n",
       "    });\n",
       "\n",
       "    // Add the css file\n",
       "    //requirejs(['d3'],\n",
       "    //    function (d3) {\n",
       "    //        d3.select('#css').attr('href', ecco_url + 'html/styles.css')\n",
       "    //    })\n",
       "\n",
       "    console.log('Ecco initialize!!')\n",
       "\n",
       "    // returns a 'basic' object. basic.init() selects the html div we'll be\n",
       "    // rendering the html into, adds styles.css to the document.\n",
       "    define('basic', ['d3'],\n",
       "        function (d3) {\n",
       "            return {\n",
       "                init: function (viz_id = null) {\n",
       "                    if (viz_id == null) {\n",
       "                        viz_id = \"viz_\" + Math.round(Math.random() * 10000000)\n",
       "                    }\n",
       "                    // Select the div rendered below, change its id\n",
       "                    const div = d3.select('#basic').attr('id', viz_id),\n",
       "                        div_parent = d3.select('#' + viz_id).node().parentNode\n",
       "\n",
       "                    // Link to CSS file\n",
       "                    d3.select(div_parent).insert('link')\n",
       "                        .attr('rel', 'stylesheet')\n",
       "                        .attr('type', 'text/css')\n",
       "                        .attr('href', ecco_url + 'html/styles.css')\n",
       "\n",
       "                    return viz_id\n",
       "                }\n",
       "            }\n",
       "        }, function (err) {\n",
       "            console.log(err);\n",
       "        }\n",
       "    )\n",
       "</script>\n",
       "\n",
       "<head>\n",
       "    <link id='css' rel=\"stylesheet\" type=\"text/css\">\n",
       "</head>\n",
       "<div id=\"basic\"></div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "         requirejs(['basic', 'ecco'], function(basic, ecco){\n",
       "            const viz_id = basic.init()\n",
       "            \n",
       "            ecco.interactiveTokensAndFactorSparklines(viz_id, {'tokens': [{'token': '[CLS]', 'token_id': 1, 'type': 'input', 'position': 0}, {'token': '▁Now', 'token_id': 790, 'type': 'input', 'position': 1}, {'token': '▁I', 'token_id': 273, 'type': 'input', 'position': 2}, {'token': '▁ask', 'token_id': 1024, 'type': 'input', 'position': 3}, {'token': '▁you', 'token_id': 274, 'type': 'input', 'position': 4}, {'token': ':', 'token_id': 294, 'type': 'input', 'position': 5}, {'token': '▁what', 'token_id': 339, 'type': 'input', 'position': 6}, {'token': '▁can', 'token_id': 295, 'type': 'input', 'position': 7}, {'token': '▁be', 'token_id': 282, 'type': 'input', 'position': 8}, {'token': '▁expected', 'token_id': 1256, 'type': 'input', 'position': 9}, {'token': '▁of', 'token_id': 265, 'type': 'input', 'position': 10}, {'token': '▁man', 'token_id': 642, 'type': 'input', 'position': 11}, {'token': '▁since', 'token_id': 515, 'type': 'input', 'position': 12}, {'token': '▁he', 'token_id': 313, 'type': 'input', 'position': 13}, {'token': '▁is', 'token_id': 269, 'type': 'input', 'position': 14}, {'token': '▁a', 'token_id': 266, 'type': 'input', 'position': 15}, {'token': '▁being', 'token_id': 411, 'type': 'input', 'position': 16}, {'token': '▁endowed', 'token_id': 29344, 'type': 'input', 'position': 17}, {'token': '▁with', 'token_id': 275, 'type': 'input', 'position': 18}, {'token': '▁strange', 'token_id': 3981, 'type': 'input', 'position': 19}, {'token': '▁qualities', 'token_id': 6686, 'type': 'input', 'position': 20}, {'token': '?', 'token_id': 302, 'type': 'input', 'position': 21}, {'token': '▁Shower', 'token_id': 12586, 'type': 'input', 'position': 22}, {'token': '▁upon', 'token_id': 1064, 'type': 'input', 'position': 23}, {'token': '▁him', 'token_id': 417, 'type': 'input', 'position': 24}, {'token': '▁every', 'token_id': 469, 'type': 'input', 'position': 25}, {'token': '▁earthly', 'token_id': 24001, 'type': 'input', 'position': 26}, {'token': '▁blessing', 'token_id': 8348, 'type': 'input', 'position': 27}, {'token': ',', 'token_id': 261, 'type': 'input', 'position': 28}, {'token': '▁drown', 'token_id': 27126, 'type': 'input', 'position': 29}, {'token': '▁him', 'token_id': 417, 'type': 'input', 'position': 30}, {'token': '▁in', 'token_id': 267, 'type': 'input', 'position': 31}, {'token': '▁a', 'token_id': 266, 'type': 'input', 'position': 32}, {'token': '▁sea', 'token_id': 2164, 'type': 'input', 'position': 33}, {'token': '▁of', 'token_id': 265, 'type': 'input', 'position': 34}, {'token': '▁happiness', 'token_id': 5116, 'type': 'input', 'position': 35}, {'token': ',', 'token_id': 261, 'type': 'input', 'position': 36}, {'token': '▁so', 'token_id': 324, 'type': 'input', 'position': 37}, {'token': '▁that', 'token_id': 272, 'type': 'input', 'position': 38}, {'token': '▁nothing', 'token_id': 942, 'type': 'input', 'position': 39}, {'token': '▁but', 'token_id': 304, 'type': 'input', 'position': 40}, {'token': '▁bubbles', 'token_id': 14007, 'type': 'input', 'position': 41}, {'token': '▁of', 'token_id': 265, 'type': 'input', 'position': 42}, {'token': '▁bliss', 'token_id': 20531, 'type': 'input', 'position': 43}, {'token': '▁can', 'token_id': 295, 'type': 'input', 'position': 44}, {'token': '▁be', 'token_id': 282, 'type': 'input', 'position': 45}, {'token': '▁seen', 'token_id': 757, 'type': 'input', 'position': 46}, {'token': '▁on', 'token_id': 277, 'type': 'input', 'position': 47}, {'token': '▁the', 'token_id': 262, 'type': 'input', 'position': 48}, {'token': '▁surface', 'token_id': 1694, 'type': 'input', 'position': 49}, {'token': ';', 'token_id': 346, 'type': 'input', 'position': 50}, {'token': '▁give', 'token_id': 527, 'type': 'input', 'position': 51}, {'token': '▁him', 'token_id': 417, 'type': 'input', 'position': 52}, {'token': '▁economic', 'token_id': 1505, 'type': 'input', 'position': 53}, {'token': '▁prosperity', 'token_id': 11779, 'type': 'input', 'position': 54}, {'token': ',', 'token_id': 261, 'type': 'input', 'position': 55}, {'token': '▁such', 'token_id': 405, 'type': 'input', 'position': 56}, {'token': '▁that', 'token_id': 272, 'type': 'input', 'position': 57}, {'token': '▁he', 'token_id': 313, 'type': 'input', 'position': 58}, {'token': '▁should', 'token_id': 403, 'type': 'input', 'position': 59}, {'token': '▁have', 'token_id': 286, 'type': 'input', 'position': 60}, {'token': '▁nothing', 'token_id': 942, 'type': 'input', 'position': 61}, {'token': '▁else', 'token_id': 995, 'type': 'input', 'position': 62}, {'token': '▁to', 'token_id': 264, 'type': 'input', 'position': 63}, {'token': '▁do', 'token_id': 333, 'type': 'input', 'position': 64}, {'token': '▁but', 'token_id': 304, 'type': 'input', 'position': 65}, {'token': '▁sleep', 'token_id': 1873, 'type': 'input', 'position': 66}, {'token': ',', 'token_id': 261, 'type': 'input', 'position': 67}, {'token': '▁eat', 'token_id': 1672, 'type': 'input', 'position': 68}, {'token': '▁cakes', 'token_id': 9582, 'type': 'input', 'position': 69}, {'token': '▁and', 'token_id': 263, 'type': 'input', 'position': 70}, {'token': '▁busy', 'token_id': 2418, 'type': 'input', 'position': 71}, {'token': '▁himself', 'token_id': 1385, 'type': 'input', 'position': 72}, {'token': '▁with', 'token_id': 275, 'type': 'input', 'position': 73}, {'token': '▁the', 'token_id': 262, 'type': 'input', 'position': 74}, {'token': '▁continuation', 'token_id': 15173, 'type': 'input', 'position': 75}, {'token': '▁of', 'token_id': 265, 'type': 'input', 'position': 76}, {'token': '▁his', 'token_id': 315, 'type': 'input', 'position': 77}, {'token': '▁species', 'token_id': 2260, 'type': 'input', 'position': 78}, {'token': ',', 'token_id': 261, 'type': 'input', 'position': 79}, {'token': '▁and', 'token_id': 263, 'type': 'input', 'position': 80}, {'token': '▁even', 'token_id': 402, 'type': 'input', 'position': 81}, {'token': '▁then', 'token_id': 393, 'type': 'input', 'position': 82}, {'token': '▁out', 'token_id': 321, 'type': 'input', 'position': 83}, {'token': '▁of', 'token_id': 265, 'type': 'input', 'position': 84}, {'token': '▁sheer', 'token_id': 8419, 'type': 'input', 'position': 85}, {'token': '▁in', 'token_id': 267, 'type': 'input', 'position': 86}, {'token': 'gratitude', 'token_id': 113691, 'type': 'input', 'position': 87}, {'token': ',', 'token_id': 261, 'type': 'input', 'position': 88}, {'token': '▁sheer', 'token_id': 8419, 'type': 'input', 'position': 89}, {'token': '▁spite', 'token_id': 9239, 'type': 'input', 'position': 90}, {'token': ',', 'token_id': 261, 'type': 'input', 'position': 91}, {'token': '▁man', 'token_id': 642, 'type': 'input', 'position': 92}, {'token': '▁would', 'token_id': 338, 'type': 'input', 'position': 93}, {'token': '▁play', 'token_id': 612, 'type': 'input', 'position': 94}, {'token': '▁you', 'token_id': 274, 'type': 'input', 'position': 95}, {'token': '▁some', 'token_id': 347, 'type': 'input', 'position': 96}, {'token': '▁nasty', 'token_id': 10211, 'type': 'input', 'position': 97}, {'token': '▁trick', 'token_id': 5999, 'type': 'input', 'position': 98}, {'token': '.', 'token_id': 260, 'type': 'input', 'position': 99}, {'token': '▁He', 'token_id': 383, 'type': 'input', 'position': 100}, {'token': '▁would', 'token_id': 338, 'type': 'input', 'position': 101}, {'token': '▁even', 'token_id': 402, 'type': 'input', 'position': 102}, {'token': '▁risk', 'token_id': 1051, 'type': 'input', 'position': 103}, {'token': '▁his', 'token_id': 315, 'type': 'input', 'position': 104}, {'token': '▁cakes', 'token_id': 9582, 'type': 'input', 'position': 105}, {'token': '▁and', 'token_id': 263, 'type': 'input', 'position': 106}, {'token': '▁would', 'token_id': 338, 'type': 'input', 'position': 107}, {'token': '▁deliberately', 'token_id': 11544, 'type': 'input', 'position': 108}, {'token': '▁desire', 'token_id': 2629, 'type': 'input', 'position': 109}, {'token': '▁the', 'token_id': 262, 'type': 'input', 'position': 110}, {'token': '▁most', 'token_id': 370, 'type': 'input', 'position': 111}, {'token': '▁fatal', 'token_id': 9329, 'type': 'input', 'position': 112}, {'token': '▁rubbish', 'token_id': 10775, 'type': 'input', 'position': 113}, {'token': ',', 'token_id': 261, 'type': 'input', 'position': 114}, {'token': '▁the', 'token_id': 262, 'type': 'input', 'position': 115}, {'token': '▁most', 'token_id': 370, 'type': 'input', 'position': 116}, {'token': '▁un', 'token_id': 1655, 'type': 'input', 'position': 117}, {'token': 'economic', 'token_id': 13647, 'type': 'input', 'position': 118}, {'token': 'al', 'token_id': 1544, 'type': 'input', 'position': 119}, {'token': '▁absurdity', 'token_id': 40855, 'type': 'input', 'position': 120}, {'token': ',', 'token_id': 261, 'type': 'input', 'position': 121}, {'token': '▁simply', 'token_id': 891, 'type': 'input', 'position': 122}, {'token': '▁to', 'token_id': 264, 'type': 'input', 'position': 123}, {'token': '▁introduce', 'token_id': 4506, 'type': 'input', 'position': 124}, {'token': '▁into', 'token_id': 352, 'type': 'input', 'position': 125}, {'token': '▁all', 'token_id': 305, 'type': 'input', 'position': 126}, {'token': '▁this', 'token_id': 291, 'type': 'input', 'position': 127}, {'token': '▁positive', 'token_id': 1453, 'type': 'input', 'position': 128}, {'token': '▁good', 'token_id': 397, 'type': 'input', 'position': 129}, {'token': '▁sense', 'token_id': 1100, 'type': 'input', 'position': 130}, {'token': '▁his', 'token_id': 315, 'type': 'input', 'position': 131}, {'token': '▁fatal', 'token_id': 9329, 'type': 'input', 'position': 132}, {'token': '▁fantastic', 'token_id': 2391, 'type': 'input', 'position': 133}, {'token': '▁element', 'token_id': 3036, 'type': 'input', 'position': 134}, {'token': '.', 'token_id': 260, 'type': 'input', 'position': 135}, {'token': '▁It', 'token_id': 325, 'type': 'input', 'position': 136}, {'token': '▁is', 'token_id': 269, 'type': 'input', 'position': 137}, {'token': '▁just', 'token_id': 348, 'type': 'input', 'position': 138}, {'token': '▁his', 'token_id': 315, 'type': 'input', 'position': 139}, {'token': '▁fantastic', 'token_id': 2391, 'type': 'input', 'position': 140}, {'token': '▁dreams', 'token_id': 3968, 'type': 'input', 'position': 141}, {'token': ',', 'token_id': 261, 'type': 'input', 'position': 142}, {'token': '▁his', 'token_id': 315, 'type': 'input', 'position': 143}, {'token': '▁vulgar', 'token_id': 33706, 'type': 'input', 'position': 144}, {'token': '▁folly', 'token_id': 38699, 'type': 'input', 'position': 145}, {'token': '▁that', 'token_id': 272, 'type': 'input', 'position': 146}, {'token': '▁he', 'token_id': 313, 'type': 'input', 'position': 147}, {'token': '▁will', 'token_id': 296, 'type': 'input', 'position': 148}, {'token': '▁desire', 'token_id': 2629, 'type': 'input', 'position': 149}, {'token': '▁to', 'token_id': 264, 'type': 'input', 'position': 150}, {'token': '▁retain', 'token_id': 6044, 'type': 'input', 'position': 151}, {'token': ',', 'token_id': 261, 'type': 'input', 'position': 152}, {'token': '▁simply', 'token_id': 891, 'type': 'input', 'position': 153}, {'token': '▁in', 'token_id': 267, 'type': 'input', 'position': 154}, {'token': '▁order', 'token_id': 556, 'type': 'input', 'position': 155}, {'token': '▁to', 'token_id': 264, 'type': 'input', 'position': 156}, {'token': '▁prove', 'token_id': 3303, 'type': 'input', 'position': 157}, {'token': '▁to', 'token_id': 264, 'type': 'input', 'position': 158}, {'token': '▁himself', 'token_id': 1385, 'type': 'input', 'position': 159}, {'token': '-', 'token_id': 271, 'type': 'input', 'position': 160}, {'token': '-', 'token_id': 271, 'type': 'input', 'position': 161}, {'token': 'as', 'token_id': 1628, 'type': 'input', 'position': 162}, {'token': '▁though', 'token_id': 651, 'type': 'input', 'position': 163}, {'token': '▁that', 'token_id': 272, 'type': 'input', 'position': 164}, {'token': '▁were', 'token_id': 332, 'type': 'input', 'position': 165}, {'token': '▁so', 'token_id': 324, 'type': 'input', 'position': 166}, {'token': '▁necessary', 'token_id': 1241, 'type': 'input', 'position': 167}, {'token': '-', 'token_id': 271, 'type': 'input', 'position': 168}, {'token': '-', 'token_id': 271, 'type': 'input', 'position': 169}, {'token': '▁that', 'token_id': 272, 'type': 'input', 'position': 170}, {'token': '▁men', 'token_id': 842, 'type': 'input', 'position': 171}, {'token': '▁still', 'token_id': 449, 'type': 'input', 'position': 172}, {'token': '▁are', 'token_id': 281, 'type': 'input', 'position': 173}, {'token': '▁men', 'token_id': 842, 'type': 'input', 'position': 174}, {'token': '▁and', 'token_id': 263, 'type': 'input', 'position': 175}, {'token': '▁not', 'token_id': 298, 'type': 'input', 'position': 176}, {'token': '▁the', 'token_id': 262, 'type': 'input', 'position': 177}, {'token': '▁keys', 'token_id': 5258, 'type': 'input', 'position': 178}, {'token': '▁of', 'token_id': 265, 'type': 'input', 'position': 179}, {'token': '▁a', 'token_id': 266, 'type': 'input', 'position': 180}, {'token': '▁piano', 'token_id': 5914, 'type': 'input', 'position': 181}, {'token': ',', 'token_id': 261, 'type': 'input', 'position': 182}, {'token': '▁which', 'token_id': 319, 'type': 'input', 'position': 183}, {'token': '▁the', 'token_id': 262, 'type': 'input', 'position': 184}, {'token': '▁laws', 'token_id': 2326, 'type': 'input', 'position': 185}, {'token': '▁of', 'token_id': 265, 'type': 'input', 'position': 186}, {'token': '▁nature', 'token_id': 1417, 'type': 'input', 'position': 187}, {'token': '▁threaten', 'token_id': 13494, 'type': 'input', 'position': 188}, {'token': '▁to', 'token_id': 264, 'type': 'input', 'position': 189}, {'token': '▁control', 'token_id': 719, 'type': 'input', 'position': 190}, {'token': '▁so', 'token_id': 324, 'type': 'input', 'position': 191}, {'token': '▁completely', 'token_id': 1298, 'type': 'input', 'position': 192}, {'token': '▁that', 'token_id': 272, 'type': 'input', 'position': 193}, {'token': '▁soon', 'token_id': 950, 'type': 'input', 'position': 194}, {'token': '▁one', 'token_id': 311, 'type': 'input', 'position': 195}, {'token': '▁will', 'token_id': 296, 'type': 'input', 'position': 196}, {'token': '▁be', 'token_id': 282, 'type': 'input', 'position': 197}, {'token': '▁able', 'token_id': 526, 'type': 'input', 'position': 198}, {'token': '▁to', 'token_id': 264, 'type': 'input', 'position': 199}, {'token': '▁desire', 'token_id': 2629, 'type': 'input', 'position': 200}, {'token': '▁nothing', 'token_id': 942, 'type': 'input', 'position': 201}, {'token': '▁but', 'token_id': 304, 'type': 'input', 'position': 202}, {'token': '▁by', 'token_id': 293, 'type': 'input', 'position': 203}, {'token': '▁the', 'token_id': 262, 'type': 'input', 'position': 204}, {'token': '▁calendar', 'token_id': 4259, 'type': 'input', 'position': 205}, {'token': '.', 'token_id': 260, 'type': 'input', 'position': 206}, {'token': '▁And', 'token_id': 414, 'type': 'input', 'position': 207}, {'token': '▁that', 'token_id': 272, 'type': 'input', 'position': 208}, {'token': '▁is', 'token_id': 269, 'type': 'input', 'position': 209}, {'token': '▁not', 'token_id': 298, 'type': 'input', 'position': 210}, {'token': '▁all', 'token_id': 305, 'type': 'input', 'position': 211}, {'token': ':', 'token_id': 294, 'type': 'input', 'position': 212}, {'token': '▁even', 'token_id': 402, 'type': 'input', 'position': 213}, {'token': '▁if', 'token_id': 337, 'type': 'input', 'position': 214}, {'token': '▁man', 'token_id': 642, 'type': 'input', 'position': 215}, {'token': '▁really', 'token_id': 431, 'type': 'input', 'position': 216}, {'token': '▁were', 'token_id': 332, 'type': 'input', 'position': 217}, {'token': '▁nothing', 'token_id': 942, 'type': 'input', 'position': 218}, {'token': '▁but', 'token_id': 304, 'type': 'input', 'position': 219}, {'token': '▁a', 'token_id': 266, 'type': 'input', 'position': 220}, {'token': '▁piano', 'token_id': 5914, 'type': 'input', 'position': 221}, {'token': '-', 'token_id': 271, 'type': 'input', 'position': 222}, {'token': 'key', 'token_id': 7871, 'type': 'input', 'position': 223}, {'token': ',', 'token_id': 261, 'type': 'input', 'position': 224}, {'token': '▁even', 'token_id': 402, 'type': 'input', 'position': 225}, {'token': '▁if', 'token_id': 337, 'type': 'input', 'position': 226}, {'token': '▁this', 'token_id': 291, 'type': 'input', 'position': 227}, {'token': '▁were', 'token_id': 332, 'type': 'input', 'position': 228}, {'token': '▁proved', 'token_id': 4776, 'type': 'input', 'position': 229}, {'token': '▁to', 'token_id': 264, 'type': 'input', 'position': 230}, {'token': '▁him', 'token_id': 417, 'type': 'input', 'position': 231}, {'token': '▁by', 'token_id': 293, 'type': 'input', 'position': 232}, {'token': '▁natural', 'token_id': 1008, 'type': 'input', 'position': 233}, {'token': '▁science', 'token_id': 1693, 'type': 'input', 'position': 234}, {'token': '▁and', 'token_id': 263, 'type': 'input', 'position': 235}, {'token': '▁mathematics', 'token_id': 9528, 'type': 'input', 'position': 236}, {'token': ',', 'token_id': 261, 'type': 'input', 'position': 237}, {'token': '▁even', 'token_id': 402, 'type': 'input', 'position': 238}, {'token': '▁then', 'token_id': 393, 'type': 'input', 'position': 239}, {'token': '▁he', 'token_id': 313, 'type': 'input', 'position': 240}, {'token': '▁would', 'token_id': 338, 'type': 'input', 'position': 241}, {'token': '▁not', 'token_id': 298, 'type': 'input', 'position': 242}, {'token': '▁become', 'token_id': 638, 'type': 'input', 'position': 243}, {'token': '▁reasonable', 'token_id': 3092, 'type': 'input', 'position': 244}, {'token': ',', 'token_id': 261, 'type': 'input', 'position': 245}, {'token': '▁but', 'token_id': 304, 'type': 'input', 'position': 246}, {'token': '▁would', 'token_id': 338, 'type': 'input', 'position': 247}, {'token': '▁purposely', 'token_id': 29203, 'type': 'input', 'position': 248}, {'token': '▁do', 'token_id': 333, 'type': 'input', 'position': 249}, {'token': '▁something', 'token_id': 491, 'type': 'input', 'position': 250}, {'token': '▁perverse', 'token_id': 38894, 'type': 'input', 'position': 251}, {'token': '▁out', 'token_id': 321, 'type': 'input', 'position': 252}, {'token': '▁of', 'token_id': 265, 'type': 'input', 'position': 253}, {'token': '▁simple', 'token_id': 860, 'type': 'input', 'position': 254}, {'token': '▁in', 'token_id': 267, 'type': 'input', 'position': 255}, {'token': 'gratitude', 'token_id': 113691, 'type': 'input', 'position': 256}, {'token': ',', 'token_id': 261, 'type': 'input', 'position': 257}, {'token': '▁simply', 'token_id': 891, 'type': 'input', 'position': 258}, {'token': '▁to', 'token_id': 264, 'type': 'input', 'position': 259}, {'token': '▁gain', 'token_id': 2088, 'type': 'input', 'position': 260}, {'token': '▁his', 'token_id': 315, 'type': 'input', 'position': 261}, {'token': '▁point', 'token_id': 582, 'type': 'input', 'position': 262}, {'token': '.', 'token_id': 260, 'type': 'input', 'position': 263}, {'token': '▁And', 'token_id': 414, 'type': 'input', 'position': 264}, {'token': '▁if', 'token_id': 337, 'type': 'input', 'position': 265}, {'token': '▁he', 'token_id': 313, 'type': 'input', 'position': 266}, {'token': '▁does', 'token_id': 490, 'type': 'input', 'position': 267}, {'token': '▁not', 'token_id': 298, 'type': 'input', 'position': 268}, {'token': '▁find', 'token_id': 433, 'type': 'input', 'position': 269}, {'token': '▁means', 'token_id': 666, 'type': 'input', 'position': 270}, {'token': '▁he', 'token_id': 313, 'type': 'input', 'position': 271}, {'token': '▁will', 'token_id': 296, 'type': 'input', 'position': 272}, {'token': '▁contrive', 'token_id': 116208, 'type': 'input', 'position': 273}, {'token': '▁destruction', 'token_id': 6138, 'type': 'input', 'position': 274}, {'token': '▁and', 'token_id': 263, 'type': 'input', 'position': 275}, {'token': '▁chaos', 'token_id': 8252, 'type': 'input', 'position': 276}, {'token': ',', 'token_id': 261, 'type': 'input', 'position': 277}, {'token': '▁will', 'token_id': 296, 'type': 'input', 'position': 278}, {'token': '▁contrive', 'token_id': 116208, 'type': 'input', 'position': 279}, {'token': '▁sufferings', 'token_id': 58292, 'type': 'input', 'position': 280}, {'token': '▁of', 'token_id': 265, 'type': 'input', 'position': 281}, {'token': '▁all', 'token_id': 305, 'type': 'input', 'position': 282}, {'token': '▁sorts', 'token_id': 5525, 'type': 'input', 'position': 283}, {'token': ',', 'token_id': 261, 'type': 'input', 'position': 284}, {'token': '▁only', 'token_id': 364, 'type': 'input', 'position': 285}, {'token': '▁to', 'token_id': 264, 'type': 'input', 'position': 286}, {'token': '▁gain', 'token_id': 2088, 'type': 'input', 'position': 287}, {'token': '▁his', 'token_id': 315, 'type': 'input', 'position': 288}, {'token': '▁point', 'token_id': 582, 'type': 'input', 'position': 289}, {'token': '!', 'token_id': 300, 'type': 'input', 'position': 290}, {'token': '▁He', 'token_id': 383, 'type': 'input', 'position': 291}, {'token': '▁will', 'token_id': 296, 'type': 'input', 'position': 292}, {'token': '▁launch', 'token_id': 2529, 'type': 'input', 'position': 293}, {'token': '▁a', 'token_id': 266, 'type': 'input', 'position': 294}, {'token': '▁curse', 'token_id': 14417, 'type': 'input', 'position': 295}, {'token': '▁upon', 'token_id': 1064, 'type': 'input', 'position': 296}, {'token': '▁the', 'token_id': 262, 'type': 'input', 'position': 297}, {'token': '▁world', 'token_id': 447, 'type': 'input', 'position': 298}, {'token': ',', 'token_id': 261, 'type': 'input', 'position': 299}, {'token': '▁and', 'token_id': 263, 'type': 'input', 'position': 300}, {'token': '▁as', 'token_id': 283, 'type': 'input', 'position': 301}, {'token': '▁only', 'token_id': 364, 'type': 'input', 'position': 302}, {'token': '▁man', 'token_id': 642, 'type': 'input', 'position': 303}, {'token': '▁can', 'token_id': 295, 'type': 'input', 'position': 304}, {'token': '▁curse', 'token_id': 14417, 'type': 'input', 'position': 305}, {'token': '▁(', 'token_id': 287, 'type': 'input', 'position': 306}, {'token': 'it', 'token_id': 1632, 'type': 'input', 'position': 307}, {'token': '▁is', 'token_id': 269, 'type': 'input', 'position': 308}, {'token': '▁his', 'token_id': 315, 'type': 'input', 'position': 309}, {'token': '▁privilege', 'token_id': 7668, 'type': 'input', 'position': 310}, {'token': ',', 'token_id': 261, 'type': 'input', 'position': 311}, {'token': '▁the', 'token_id': 262, 'type': 'input', 'position': 312}, {'token': '▁primary', 'token_id': 1862, 'type': 'input', 'position': 313}, {'token': '▁distinction', 'token_id': 8537, 'type': 'input', 'position': 314}, {'token': '▁between', 'token_id': 457, 'type': 'input', 'position': 315}, {'token': '▁him', 'token_id': 417, 'type': 'input', 'position': 316}, {'token': '▁and', 'token_id': 263, 'type': 'input', 'position': 317}, {'token': '▁other', 'token_id': 340, 'type': 'input', 'position': 318}, {'token': '▁animals', 'token_id': 2232, 'type': 'input', 'position': 319}, {'token': ')', 'token_id': 285, 'type': 'input', 'position': 320}, {'token': ',', 'token_id': 261, 'type': 'input', 'position': 321}, {'token': '▁may', 'token_id': 372, 'type': 'input', 'position': 322}, {'token': '▁be', 'token_id': 282, 'type': 'input', 'position': 323}, {'token': '▁by', 'token_id': 293, 'type': 'input', 'position': 324}, {'token': '▁his', 'token_id': 315, 'type': 'input', 'position': 325}, {'token': '▁curse', 'token_id': 14417, 'type': 'input', 'position': 326}, {'token': '▁alone', 'token_id': 1585, 'type': 'input', 'position': 327}, {'token': '▁he', 'token_id': 313, 'type': 'input', 'position': 328}, {'token': '▁will', 'token_id': 296, 'type': 'input', 'position': 329}, {'token': '▁attain', 'token_id': 11376, 'type': 'input', 'position': 330}, {'token': '▁his', 'token_id': 315, 'type': 'input', 'position': 331}, {'token': '▁object', 'token_id': 2713, 'type': 'input', 'position': 332}, {'token': '-', 'token_id': 271, 'type': 'input', 'position': 333}, {'token': '-', 'token_id': 271, 'type': 'input', 'position': 334}, {'token': 'that', 'token_id': 3727, 'type': 'input', 'position': 335}, {'token': '▁is', 'token_id': 269, 'type': 'input', 'position': 336}, {'token': ',', 'token_id': 261, 'type': 'input', 'position': 337}, {'token': '▁convince', 'token_id': 7599, 'type': 'input', 'position': 338}, {'token': '▁himself', 'token_id': 1385, 'type': 'input', 'position': 339}, {'token': '▁that', 'token_id': 272, 'type': 'input', 'position': 340}, {'token': '▁he', 'token_id': 313, 'type': 'input', 'position': 341}, {'token': '▁is', 'token_id': 269, 'type': 'input', 'position': 342}, {'token': '▁a', 'token_id': 266, 'type': 'input', 'position': 343}, {'token': '▁man', 'token_id': 642, 'type': 'input', 'position': 344}, {'token': '▁and', 'token_id': 263, 'type': 'input', 'position': 345}, {'token': '▁not', 'token_id': 298, 'type': 'input', 'position': 346}, {'token': '▁a', 'token_id': 266, 'type': 'input', 'position': 347}, {'token': '▁piano', 'token_id': 5914, 'type': 'input', 'position': 348}, {'token': '-', 'token_id': 271, 'type': 'input', 'position': 349}, {'token': 'key', 'token_id': 7871, 'type': 'input', 'position': 350}, {'token': '!', 'token_id': 300, 'type': 'input', 'position': 351}, {'token': '[SEP]', 'token_id': 2, 'type': 'input', 'position': 352}], 'factors': [[[0.005744734313338995, 0.8357282876968384, 0.3182890713214874, 0.3207283318042755, 0.15836893022060394, 0.31125307083129883, 0.5133938193321228, 0.43808287382125854, 0.3857623338699341, 0.1851053684949875, 0.09099273383617401, 0.0, 0.6882473230361938, 0.02260560169816017, 0.2119835466146469, 0.0, 0.10366227477788925, 0.04654772952198982, 0.0, 0.10016841441392899, 0.07800532132387161, 0.16889774799346924, 0.06116092577576637, 0.0, 0.004045172594487667, 0.26189079880714417, 0.0, 0.0, 0.3784893751144409, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.3239593803882599, 0.6256011724472046, 0.6617590188980103, 0.34090641140937805, 0.40504658222198486, 0.0, 0.0, 0.0, 0.30560192465782166, 0.22818031907081604, 0.0, 0.0, 0.0, 0.0, 0.42648768424987793, 0.06756454706192017, 0.0004664076550398022, 0.0024196170270442963, 0.0, 0.44197073578834534, 0.7234419584274292, 0.6704423427581787, 0.01543527003377676, 0.4157666563987732, 0.0, 0.21283002197742462, 0.3082924783229828, 0.0, 0.0, 0.42846056818962097, 0.0, 0.2779485881328583, 0.0, 0.0, 0.42269647121429443, 0.0, 0.12644679844379425, 0.0, 0.0014507989399135113, 0.07100411504507065, 0.02305559068918228, 0.0, 0.03786744549870491, 0.31618544459342957, 0.8174585700035095, 0.8283249735832214, 0.7282881140708923, 0.23560209572315216, 0.23522090911865234, 0.2895599603652954, 0.0, 0.11426085978746414, 0.3404224216938019, 0.26224398612976074, 0.253531277179718, 0.3680737316608429, 0.026694316416978836, 0.34515005350112915, 0.0, 0.0, 0.3842868208885193, 0.0, 0.03994105011224747, 0.015562713146209717, 0.2278723120689392, 0.37597382068634033, 0.7424728274345398, 0.019388630986213684, 0.0, 0.0, 0.6295621991157532, 0.38658833503723145, 0.4125893712043762, 0.028673365712165833, 0.16609077155590057, 0.3093726933002472, 0.011900657787919044, 0.0161574799567461, 0.3866131901741028, 0.24264422059059143, 0.3235785663127899, 0.0074303606525063515, 0.0017041363753378391, 0.11777425557374954, 0.24004602432250977, 0.45501264929771423, 0.6921816468238831, 0.2743849754333496, 0.08947275578975677, 0.04480796307325363, 0.3346928656101227, 0.5047802329063416, 0.059001922607421875, 0.0, 0.10818116366863251, 0.0, 0.029118681326508522, 0.020987723022699356, 0.17597593367099762, 0.01904442347586155, 0.699398398399353, 0.5945611000061035, 0.5722689032554626, 0.0, 0.05055781081318855, 0.0, 0.4262961745262146, 0.00867338664829731, 0.060696955770254135, 0.12037921696901321, 0.6838046312332153, 0.06769461929798126, 0.36465591192245483, 0.15849237143993378, 0.0797242596745491, 0.042456790804862976, 0.40038710832595825, 0.6806330680847168, 0.20991376042366028, 0.48974108695983887, 0.24365229904651642, 0.16759638488292694, 0.0, 0.12982402741909027, 0.23039644956588745, 0.2586303651332855, 0.5889893174171448, 0.7200008034706116, 0.6710394620895386, 0.2955600619316101, 0.5952044725418091, 0.2733179032802582, 0.18057377636432648, 0.28059712052345276, 0.5505045056343079, 0.0, 0.40115129947662354, 0.08975593745708466, 0.0, 0.49803170561790466, 0.3497593402862549, 0.0, 0.0, 0.038444191217422485, 0.0, 0.0, 0.3884531855583191, 0.747657060623169, 0.03152104467153549, 0.03366285562515259, 0.04547155648469925, 0.030894871801137924, 0.2741791903972626, 0.10269960016012192, 0.0, 0.5532557368278503, 0.3243798613548279, 0.670781672000885, 0.4314451813697815, 0.22691667079925537, 0.3536662459373474, 0.2987114489078522, 0.264969140291214, 0.044357780367136, 0.0, 0.21260422468185425, 0.487850159406662, 0.06830897182226181, 0.0, 0.031916916370391846, 0.011518445797264576, 0.9785593748092651, 0.8113246560096741, 0.5560950636863708, 0.4690769910812378, 0.4575224816799164, 0.45046350359916687, 0.8537983894348145, 0.550597608089447, 0.0, 0.5334765911102295, 0.1965928077697754, 0.22471116483211517, 0.38586682081222534, 0.0, 0.0, 0.0, 0.003194439224898815, 0.44776299595832825, 0.8073412775993347, 0.5577382445335388, 0.6121224761009216, 0.32447516918182373, 0.1583971381187439, 0.0, 0.0006290111923590302, 0.1483629196882248, 0.0, 0.0, 0.2774271070957184, 0.050622664391994476, 0.432693749666214, 0.805866539478302, 0.6997650265693665, 0.06922481209039688, 0.3611368238925934, 0.33536770939826965, 0.17614693939685822, 0.10850923508405685, 0.352573037147522, 0.8410932421684265, 0.38849765062332153, 0.39542731642723083, 0.0, 0.39566653966903687, 0.13169212639331818, 0.18993809819221497, 0.1884613037109375, 0.20409396290779114, 0.0, 0.07423727959394455, 0.4357885718345642, 0.6294682621955872, 0.18345916271209717, 0.0, 0.0, 0.05939893424510956, 0.017692584544420242, 1.0205223560333252, 0.6015840172767639, 0.09671682864427567, 0.38938432931900024, 0.3378208577632904, 0.16149497032165527, 0.3099384903907776, 0.07503926753997803, 0.277154803276062, 0.13213704526424408, 0.0, 0.25055548548698425, 0.0, 0.3801770806312561, 0.3244044780731201, 0.12866146862506866, 0.0, 0.11107022315263748, 0.29619506001472473, 0.30626943707466125, 0.4118947386741638, 0.7075703740119934, 0.23890766501426697, 0.0, 0.0, 0.06774339824914932, 0.03910161182284355, 0.19284895062446594, 0.37262776494026184, 0.03975832462310791, 0.0, 0.0, 0.0, 0.0, 0.022671110928058624, 0.34234005212783813, 0.8035622835159302, 0.5535299181938171, 0.484134316444397, 0.0, 0.2744687795639038, 0.0, 0.2138769030570984, 0.5539968609809875, 0.4950980842113495, 0.0, 0.17592079937458038, 0.43128907680511475, 0.23881949484348297, 0.3141467869281769, 0.2480372190475464, 0.1678306758403778, 0.0, 0.2036685049533844, 0.1329115927219391, 0.0, 0.06518401205539703, 0.46954622864723206, 0.7222583889961243, 0.5483866930007935, 0.20091840624809265, 0.0, 0.003940580878406763, 0.40709659457206726, 0.05011732503771782, 0.27333343029022217, 0.10519406944513321, 0.0, 0.1403834968805313, 0.24343420565128326, 0.289007306098938, 0.7148347496986389, 0.607158899307251, 0.42219483852386475, 0.17996811866760254, 0.21288499236106873, 0.46281003952026367, 0.0, 0.10731241106987, 0.0, 0.0, 0.44298192858695984, 0.27671900391578674, 0.0, 0.0, 0.0, 5.379054709919728e-05, 0.005796958692371845, 0.0008937095990404487], [0.08279426395893097, 0.0, 0.0, 0.25967296957969666, 0.04002653434872627, 0.08122076839208603, 0.14727824926376343, 0.016772592440247536, 0.06784041225910187, 0.3731933534145355, 0.03915773704648018, 0.12339852005243301, 0.0, 0.08340651541948318, 0.0, 0.0, 0.2987810969352722, 0.25894924998283386, 0.0, 0.0, 0.7070973515510559, 0.10850010067224503, 0.4358622133731842, 0.059673864394426346, 0.03841375932097435, 0.0, 0.06725231558084488, 0.6626677513122559, 0.02613808400928974, 0.17395749688148499, 0.08759273588657379, 0.0, 0.0, 0.1511501967906952, 0.0, 0.493190735578537, 0.1742284744977951, 0.2766510248184204, 0.09758781641721725, 0.0, 0.0, 0.4480574131011963, 0.0, 0.41353726387023926, 0.0, 0.11792314797639847, 0.5049680471420288, 0.0, 0.00853329710662365, 0.49930432438850403, 0.11343903839588165, 0.3898621201515198, 0.007438604719936848, 0.21199218928813934, 0.5052996873855591, 0.09476641565561295, 0.21134395897388458, 0.11603983491659164, 0.07081615179777145, 0.014679966494441032, 0.06707211583852768, 0.0, 0.07470881193876266, 0.22816556692123413, 0.7704129219055176, 0.0, 0.40345484018325806, 0.0, 0.3220653235912323, 0.5200279951095581, 0.0, 0.29238373041152954, 0.02243788354098797, 0.0, 0.01636841706931591, 0.3165232539176941, 0.0, 0.0, 0.38345950841903687, 0.2738608717918396, 0.13543926179409027, 0.0, 0.03036027029156685, 0.0, 0.0, 0.0, 0.0, 0.39661189913749695, 0.0, 0.0, 0.5046442747116089, 0.09691684693098068, 0.11054090410470963, 0.01760980300605297, 0.5647199153900146, 0.11026082187891006, 0.0, 0.014176849275827408, 0.8601245284080505, 0.11713932454586029, 0.02325921691954136, 0.0, 0.0, 0.4605218172073364, 0.0, 0.6251198649406433, 0.11479831486940384, 0.020072441548109055, 0.13192550837993622, 0.546939492225647, 0.0, 0.0, 0.036589253693819046, 0.48628145456314087, 0.0, 0.0, 0.0, 0.04050660505890846, 0.30752354860305786, 0.023895451799035072, 0.4198910593986511, 0.06086793169379234, 0.0, 0.16718550026416779, 0.5561849474906921, 0.0036121399607509375, 0.0, 0.0, 0.14839209616184235, 0.09009761363267899, 0.6828932166099548, 0.0, 0.04220699891448021, 0.12280359119176865, 0.6846621036529541, 0.13896484673023224, 0.2002076357603073, 0.0, 0.0, 0.0, 0.12687763571739197, 0.7338607311248779, 0.0, 0.0, 0.0, 0.37399861216545105, 0.08499184250831604, 0.07668381184339523, 0.0, 0.5323570966720581, 0.12967857718467712, 0.42399975657463074, 0.047631438821554184, 0.0, 0.0, 0.396297812461853, 0.12612499296665192, 0.5903964042663574, 0.12186164408922195, 0.14188873767852783, 0.04923771321773529, 0.032390471547842026, 0.10025831311941147, 0.04565247893333435, 0.127426877617836, 0.0, 0.0, 0.44077402353286743, 0.0731746256351471, 0.08687745034694672, 0.07391322404146194, 0.04741938039660454, 0.0, 0.0, 0.059381499886512756, 0.0, 0.0, 0.07804907113313675, 0.7554671764373779, 0.0, 0.0, 0.5831919312477112, 0.15735942125320435, 0.2560250163078308, 0.02113540656864643, 0.48932310938835144, 0.0, 0.27181336283683777, 0.4139282703399658, 0.11005563288927078, 0.7378484010696411, 0.015441833063960075, 0.0, 0.06321943551301956, 0.0, 0.11445793509483337, 0.0014507737942039967, 0.037988584488630295, 0.33962899446487427, 0.22800038754940033, 0.5879960656166077, 0.0, 0.0, 0.0, 0.031543128192424774, 0.5680899024009705, 0.15208911895751953, 0.03626652806997299, 0.0554509162902832, 0.0, 0.001535185263492167, 0.08565889298915863, 0.07240727543830872, 0.0045548612251877785, 0.053854800760746, 0.08302917331457138, 0.0843593180179596, 0.0, 0.0, 0.0, 0.012101946398615837, 0.30911919474601746, 0.020709821954369545, 0.61610347032547, 0.04036442190408707, 0.00450991652905941, 0.0490172803401947, 0.030516501516103745, 0.0, 0.5588821172714233, 0.14662744104862213, 0.19057202339172363, 0.0, 0.0504118837416172, 0.4840529263019562, 0.0, 0.5202016830444336, 0.0755847692489624, 0.010815363377332687, 0.055614449083805084, 0.0918707326054573, 0.0, 0.0, 0.17947852611541748, 0.429325670003891, 0.1812005192041397, 0.0074657294899225235, 0.0048402417451143265, 0.19935910403728485, 0.49386802315711975, 0.04965599998831749, 0.09832070022821426, 0.07772408425807953, 0.0, 0.0, 0.0, 0.4212941527366638, 0.022427886724472046, 0.0, 0.10008997470140457, 0.45169368386268616, 0.0, 0.8635382652282715, 0.15764033794403076, 0.03624403104186058, 0.028758162632584572, 0.027818653732538223, 0.0, 0.0, 0.44195276498794556, 0.5918257832527161, 0.06397369503974915, 0.0, 0.3975946307182312, 0.4349153935909271, 0.0, 0.2760816812515259, 0.04800694063305855, 0.015909312292933464, 0.33427324891090393, 0.44154539704322815, 0.0, 0.0, 0.18868736922740936, 0.08594406396150589, 0.0, 0.10176200419664383, 0.42928841710090637, 0.0, 0.8421501517295837, 0.13241924345493317, 0.011775246821343899, 0.0, 0.3590240180492401, 0.0, 0.4889276921749115, 0.04024850204586983, 0.029201721772551537, 0.2299388200044632, 0.19780707359313965, 0.1967936009168625, 0.0, 0.0, 0.0900593250989914, 0.02959422580897808, 0.5242637991905212, 0.02033114619553089, 0.29935720562934875, 0.0, 0.0, 0.5942871570587158, 0.0, 0.0, 0.0019059865735471249, 0.4557841718196869, 0.0, 0.14011156558990479, 0.0, 0.0, 0.37992987036705017, 0.12228397279977798, 0.12486676126718521, 0.0, 0.1929342895746231, 0.0, 0.0, 0.546220064163208, 0.028207218274474144, 0.15090863406658173, 0.028566451743245125, 0.5303046703338623, 0.0, 0.9125441908836365, 0.0887635126709938, 0.09683626890182495, 0.15066026151180267, 0.07104511559009552, 0.06327155232429504, 0.4775335490703583, 0.08677131682634354, 0.17364546656608582, 0.1731473058462143, 0.0, 0.0, 0.10128702223300934, 0.0, 0.0, 0.0, 0.31703537702560425, 0.010495394468307495, 0.6103740334510803, 0.1389089673757553, 0.07318723201751709], [0.012876582331955433, 0.05535697564482689, 0.2957881689071655, 0.2548062801361084, 0.0, 0.3395143151283264, 0.21437206864356995, 0.0532214529812336, 0.30750414729118347, 0.12521182000637054, 0.0665748193860054, 0.15890954434871674, 0.030654462054371834, 0.05095376819372177, 0.9076405763626099, 0.5458897352218628, 0.21286331117153168, 0.06810584664344788, 0.07700344175100327, 0.08981543034315109, 0.0643274337053299, 0.048521675169467926, 0.0, 0.0, 0.019451333209872246, 0.16688163578510284, 0.06843803822994232, 0.022408297285437584, 0.06780176609754562, 0.0, 0.006241436582058668, 0.2105175256729126, 0.5003757476806641, 0.010738523676991463, 0.040972523391246796, 0.06712521612644196, 0.1645594835281372, 0.1629772037267685, 0.7685269713401794, 0.12678728997707367, 0.37919703125953674, 0.034291353076696396, 0.03656630218029022, 0.0, 0.04648524150252342, 0.3680339753627777, 0.06476645171642303, 0.10760518908500671, 0.23004840314388275, 0.08122200518846512, 0.027948230504989624, 0.10641887784004211, 0.02117273397743702, 0.04805990308523178, 0.013500836677849293, 0.17929916083812714, 0.13535797595977783, 0.7531729936599731, 5.332405635272153e-05, 0.13715428113937378, 0.35170552134513855, 0.16023927927017212, 0.05644863471388817, 0.0, 0.09704716503620148, 0.3118150234222412, 0.048532288521528244, 0.051376838237047195, 0.15794430673122406, 0.0, 0.09343764930963516, 0.05302406847476959, 0.0, 0.06900087743997574, 0.403795450925827, 0.2316286861896515, 0.19216914474964142, 0.0, 0.10120879858732224, 0.06750036031007767, 0.08328467607498169, 0.0, 0.06628528982400894, 0.0, 0.0, 0.0, 0.02104806713759899, 0.0, 0.12414922565221786, 0.0, 0.0, 0.12779761850833893, 0.0952407568693161, 0.0, 0.0, 0.017164241522550583, 0.17465123534202576, 0.0, 0.0, 0.06655596941709518, 0.0, 0.0, 0.0, 0.009904902428388596, 0.0, 0.0, 0.021669235080480576, 0.0, 0.0, 0.0814078226685524, 0.2478441298007965, 0.01505839079618454, 0.0, 0.0, 0.25261548161506653, 0.2361825704574585, 0.02382853999733925, 0.09174588322639465, 0.07661940902471542, 0.03331657499074936, 0.002106853062286973, 0.04169653356075287, 0.0, 0.0, 0.03752080723643303, 0.04327446594834328, 0.16413946449756622, 0.5088576674461365, 0.0, 0.03137810528278351, 0.09427876025438309, 0.0, 0.0, 0.037138208746910095, 0.0, 0.0737651139497757, 0.048453476279973984, 0.7221368551254272, 0.08719264715909958, 0.009937116876244545, 0.04407457634806633, 0.006808230187743902, 0.2331070601940155, 0.0, 0.0, 0.007194458041340113, 0.8322924375534058, 0.0, 0.0, 0.0, 0.0, 0.1146366149187088, 0.06586571037769318, 0.0, 0.025995906442403793, 0.0, 0.0, 0.1457001268863678, 0.0, 0.0, 0.1774546504020691, 0.20089103281497955, 0.028174206614494324, 0.11031179875135422, 0.7166348099708557, 0.7172805666923523, 0.0776655301451683, 0.1453162133693695, 0.19242718815803528, 0.16615651547908783, 0.9216399192810059, 0.15519706904888153, 0.21677197515964508, 0.8829775452613831, 0.204507976770401, 0.2545628547668457, 0.30107787251472473, 0.4597901403903961, 0.06989842653274536, 0.14992240071296692, 0.4582229256629944, 0.049782510846853256, 0.15952540934085846, 0.3568105697631836, 0.38090983033180237, 0.1908671259880066, 0.11981695145368576, 0.17868556082248688, 0.01247729267925024, 0.0, 0.1302759349346161, 0.07841753214597702, 0.008471091277897358, 0.8021670579910278, 0.031890612095594406, 0.14014805853366852, 0.019700367003679276, 0.3033784329891205, 0.0, 0.0, 0.08658451586961746, 0.146442249417305, 0.40550747513771057, 0.17122195661067963, 0.3143922686576843, 0.00369284744374454, 0.07207627594470978, 0.12855717539787292, 0.6466274857521057, 0.7730607390403748, 0.1910906434059143, 0.11254363507032394, 0.23898042738437653, 0.0, 0.1344398409128189, 0.16928867995738983, 0.20950940251350403, 0.7764183282852173, 0.19744651019573212, 0.4982430934906006, 0.5670619606971741, 0.10351088643074036, 0.15421701967716217, 0.06521803140640259, 0.11371885985136032, 0.0, 0.07430389523506165, 0.4905586242675781, 0.5923008918762207, 0.16971999406814575, 0.0, 0.005429047625511885, 0.11158840358257294, 0.16970586776733398, 0.2273692488670349, 0.18349702656269073, 0.04426001012325287, 0.13226141035556793, 0.0, 0.12667062878608704, 0.0, 0.0, 0.07768413424491882, 0.1208229586482048, 0.1182766705751419, 0.062473926693201065, 0.2612192630767822, 0.0, 0.009148877114057541, 0.11107300966978073, 0.11429176479578018, 0.0, 0.0, 0.0, 0.04622409865260124, 0.022856837138533592, 0.0, 0.060317207127809525, 0.0, 0.0, 0.0, 0.0, 0.0, 0.05869177356362343, 0.03931768238544464, 0.04093039408326149, 0.0, 0.1703130602836609, 0.13099119067192078, 0.04691719636321068, 0.15283991396427155, 0.0, 0.0, 0.08213061094284058, 0.0014151756186038256, 0.12097768485546112, 0.023918243125081062, 0.06360426545143127, 0.0, 0.06642008572816849, 0.0, 0.03358083218336105, 0.031009042635560036, 0.05157924070954323, 0.07751687616109848, 0.008138793520629406, 0.0, 0.0, 0.0, 0.0, 0.034828417003154755, 0.0, 0.0, 0.0, 0.39568033814430237, 0.06995599716901779, 0.0, 0.3009635806083679, 0.13061228394508362, 0.12817499041557312, 0.15713472664356232, 0.09548628330230713, 0.1513916403055191, 0.17999431490898132, 0.04958173260092735, 0.06547047197818756, 0.33875539898872375, 0.04531676322221756, 0.8140826225280762, 0.02142331749200821, 0.0, 0.288729727268219, 0.39557865262031555, 0.11863669753074646, 0.09406911581754684, 0.11915785074234009, 0.01283387839794159, 0.2411486953496933, 0.2276940792798996, 0.1171596348285675, 0.11156058311462402, 0.0976271778345108, 0.0, 0.3246806859970093, 0.04729800671339035, 0.0, 0.022974679246544838, 0.00764255179092288, 0.0, 0.0, 0.0, 0.0, 0.0, 0.13730448484420776, 0.17745628952980042, 0.4874880313873291, 0.7128028869628906, 0.19585688412189484, 0.10372424870729446, 0.0, 0.8667394518852234, 0.05917046219110489, 1.0186235904693604, 0.5907819271087646, 0.24589328467845917, 0.24190384149551392, 0.3203960061073303, 0.540699303150177, 0.11154880374670029, 0.15983696281909943, 0.07244543731212616, 0.009278402663767338, 0.017587807029485703], [0.0, 0.07156729698181152, 0.0, 0.4802907109260559, 0.041573576629161835, 0.36749932169914246, 0.24535579979419708, 0.1284416764974594, 0.139582559466362, 0.29980775713920593, 0.24017129838466644, 0.0018199343467131257, 0.541373074054718, 0.0043507106602191925, 0.10201290994882584, 0.05642571300268173, 0.0, 0.14958934485912323, 0.7467140555381775, 0.0, 0.0, 0.24853955209255219, 0.1825157254934311, 0.6198708415031433, 0.0, 0.11637569963932037, 0.0, 0.0, 0.07135770469903946, 0.24352839589118958, 0.0, 0.9027461409568787, 0.025066666305065155, 0.010203927755355835, 0.0, 0.0, 0.013839519582688808, 0.316849023103714, 0.17311204969882965, 0.0, 0.0, 0.05764829367399216, 0.025153467431664467, 0.0, 0.16313812136650085, 0.23273253440856934, 0.21473069489002228, 0.7503320574760437, 0.018530825152993202, 0.0, 0.24498192965984344, 0.625322163105011, 0.0, 0.0, 0.0, 0.08336902409791946, 0.21397890150547028, 0.140969380736351, 0.0, 0.20460039377212524, 0.43676960468292236, 0.0, 0.0, 0.13151544332504272, 0.09377492219209671, 0.015779029577970505, 0.0, 0.059039898216724396, 0.0, 0.0, 0.004822649527341127, 0.29671111702919006, 0.0, 0.9222509264945984, 0.06590002775192261, 0.1120324581861496, 0.14899826049804688, 0.0, 0.0, 0.0, 0.001964348601177335, 0.048671361058950424, 0.050014346837997437, 0.5743775367736816, 0.44528743624687195, 0.03247556835412979, 0.1857069581747055, 0.001826736843213439, 0.01314238179475069, 0.0, 0.005988076329231262, 0.0, 0.0, 0.09826970845460892, 0.29745519161224365, 0.0, 0.04572250321507454, 0.0, 0.023500673472881317, 0.07772558182477951, 0.0, 0.09685026109218597, 0.0, 0.29416772723197937, 0.0, 0.0, 0.0, 0.03940963372588158, 0.07353364676237106, 0.25017353892326355, 0.07178701460361481, 0.0, 0.0, 0.0, 0.07314836233854294, 0.0386185385286808, 0.0, 0.0, 0.0, 0.0, 0.0, 0.05569399520754814, 0.21381764113903046, 0.4839818477630615, 0.5659000873565674, 0.8295978903770447, 0.044083405286073685, 0.0, 0.0010696918470785022, 0.0, 0.01162792183458805, 0.0, 0.0, 0.0, 0.0, 0.048657990992069244, 0.0014086375012993813, 0.21270237863063812, 0.09057790786027908, 0.0, 0.005939149763435125, 0.0, 0.15380002558231354, 0.0, 0.0, 0.0, 0.0473618321120739, 0.0, 0.0, 0.1813443899154663, 0.1470032036304474, 0.26859787106513977, 0.11397435516119003, 0.1779966652393341, 0.8443441987037659, 0.2207169234752655, 0.39930328726768494, 0.47017818689346313, 0.3841169476509094, 0.0, 0.005550360772758722, 0.0, 0.15391714870929718, 0.10694972425699234, 0.0, 0.24322105944156647, 0.0014058944070711732, 0.062874436378479, 0.0, 0.0, 0.046861495822668076, 0.0, 0.055019084364175797, 0.10898026078939438, 0.0, 0.022885169833898544, 0.17352184653282166, 0.010413933545351028, 0.0, 0.07334236800670624, 0.0, 0.0322040319442749, 0.0, 0.14956018328666687, 0.04010023921728134, 0.0, 0.025059202685952187, 0.0, 0.3243843913078308, 0.2582946717739105, 0.14623139798641205, 0.13640350103378296, 0.05984124541282654, 0.13086345791816711, 0.19678428769111633, 0.0, 0.02404409646987915, 0.2253757268190384, 0.13031888008117676, 0.17850857973098755, 0.16021336615085602, 0.0, 0.0, 0.6457402110099792, 0.0, 0.0, 0.03828120604157448, 0.035710424184799194, 0.0, 0.13339905440807343, 0.14426474273204803, 0.007623881567269564, 0.2973822355270386, 0.10600422322750092, 0.35091325640678406, 0.0, 0.0, 0.2851683795452118, 0.0, 0.011560372076928616, 0.0, 0.05064745619893074, 0.0, 0.0, 0.018037762492895126, 0.07940050959587097, 0.36836519837379456, 0.0, 0.27641764283180237, 0.44015243649482727, 0.4910147786140442, 0.0, 0.7045756578445435, 0.0, 0.0, 0.0, 0.0, 0.02557678148150444, 0.03216162696480751, 0.086452916264534, 0.0, 0.15310868620872498, 0.16643498837947845, 0.5684459805488586, 0.0, 0.0, 0.0028769259806722403, 0.11425533145666122, 0.08380922675132751, 0.37256672978401184, 0.0, 0.0, 0.5647569894790649, 0.43428611755371094, 0.00445554917678237, 0.16700021922588348, 0.0, 0.03523005172610283, 0.1278226375579834, 0.4597010612487793, 0.583533763885498, 0.0, 0.0, 0.05838314816355705, 0.007642759475857019, 0.4067881107330322, 0.0, 0.2801414728164673, 0.17406614124774933, 0.27965864539146423, 0.10474842041730881, 0.0, 0.021895775571465492, 0.29053404927253723, 0.0, 0.0, 0.0, 0.0, 0.0, 0.23075902462005615, 0.0, 0.14791254699230194, 0.0, 0.0, 0.06797454506158829, 0.17080608010292053, 0.45474374294281006, 0.5752615332603455, 0.0, 0.0, 0.15440602600574493, 0.0, 0.025305384770035744, 0.48715388774871826, 0.005006116349250078, 0.001557792886160314, 0.6295689344406128, 0.0, 0.0, 0.0, 0.0, 0.3739013671875, 0.0, 0.0, 0.0, 0.0, 0.30043381452560425, 0.0, 0.19725576043128967, 0.0, 0.0, 0.10928889364004135, 0.08562429249286652, 0.0, 0.02598395384848118, 0.5140336155891418, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0642419382929802, 0.1977144330739975, 0.7872757911682129, 0.0, 0.0006672887830063701, 0.02849045768380165, 0.0, 0.03692963719367981, 0.5341868996620178, 0.0, 0.009634037502110004, 0.0, 0.0, 0.0, 0.03924226760864258, 0.01090538501739502, 0.4137723445892334, 0.0, 0.15715108811855316, 0.0, 0.10565029829740524, 0.0, 0.0, 0.0, 0.13889455795288086, 0.0, 0.03791304677724838, 0.0, 0.0, 0.0, 0.0], [0.09642579406499863, 0.0, 0.0, 0.0, 0.0, 0.07213690876960754, 0.05517483130097389, 0.0, 0.02921055443584919, 0.007400887086987495, 0.4285922050476074, 0.1618116796016693, 0.0, 0.0030908710323274136, 0.0, 0.1519993096590042, 0.01416311040520668, 0.1763322949409485, 0.3178376853466034, 0.0, 0.08072389662265778, 0.24664439260959625, 0.10236815363168716, 0.19558364152908325, 0.21830134093761444, 0.1919412761926651, 0.20426838099956512, 0.0510108582675457, 0.31852784752845764, 0.3066690266132355, 0.21628186106681824, 0.2582239508628845, 0.25596609711647034, 0.4594278335571289, 0.7203077077865601, 0.017605405300855637, 0.3636062741279602, 0.030875302851200104, 0.0, 0.36541563272476196, 0.3236783742904663, 0.2965258061885834, 0.6890154480934143, 0.04377005994319916, 0.11235610395669937, 0.06615649908781052, 0.10306479781866074, 0.2794453203678131, 0.5129777193069458, 0.30371707677841187, 0.19764508306980133, 0.0, 0.16438628733158112, 0.20026768743991852, 0.05863353982567787, 0.3194451630115509, 0.06936901062726974, 0.0, 0.015490949153900146, 0.0038609555922448635, 0.0, 0.3639540374279022, 0.29255613684654236, 0.3549400866031647, 0.1292639970779419, 0.3610149919986725, 0.1455138623714447, 0.4419746994972229, 0.21639899909496307, 0.21541903913021088, 0.3314950168132782, 0.12847073376178741, 0.47751742601394653, 0.1856091320514679, 0.37349769473075867, 0.11257811635732651, 0.5483593940734863, 0.355445921421051, 0.071774423122406, 0.3599637448787689, 0.1038363128900528, 0.2540709376335144, 0.0025402584578841925, 0.38299524784088135, 0.5552929639816284, 0.488017201423645, 0.7120504379272461, 0.23637473583221436, 0.5046246647834778, 0.4841011166572571, 0.07058730721473694, 0.2764689326286316, 0.13431039452552795, 0.01445658877491951, 0.047948289662599564, 0.0, 0.09311390668153763, 0.044914789497852325, 0.0, 0.13087858259677887, 0.0, 0.0, 0.06217218190431595, 0.0, 0.2489113211631775, 0.12464750558137894, 0.1838691681623459, 0.007336822338402271, 0.0, 0.0, 0.4253000319004059, 0.3851487338542938, 0.19964741170406342, 0.0, 0.42428871989250183, 0.4653942883014679, 0.4250713884830475, 0.5310885906219482, 0.25416135787963867, 0.3995317816734314, 0.010643676854670048, 0.37366050481796265, 0.05607493221759796, 0.08025125414133072, 0.0, 0.25788912177085876, 0.2856660485267639, 0.08400136232376099, 0.0, 0.1499202698469162, 0.11503200232982635, 0.27614811062812805, 0.21460123360157013, 0.16238531470298767, 0.0, 0.12182488292455673, 0.0, 0.0, 0.0, 0.2415294051170349, 0.21545004844665527, 0.05766963213682175, 0.3317827880382538, 0.2943292260169983, 0.07059717178344727, 0.02451437897980213, 0.0, 0.0, 0.0, 0.0, 0.1811881810426712, 0.0, 0.34898173809051514, 0.07420161366462708, 0.2915657162666321, 0.18037033081054688, 0.1637403517961502, 0.0, 0.39066630601882935, 0.4008669853210449, 0.6951141357421875, 0.648481011390686, 0.2508712112903595, 0.04308415204286575, 0.0, 0.005179147236049175, 0.13663142919540405, 0.0, 0.6139311194419861, 0.5443257093429565, 0.0, 0.18339699506759644, 0.0943991020321846, 0.0, 0.24554185569286346, 0.30374905467033386, 0.19054649770259857, 0.32612740993499756, 0.19980457425117493, 0.6254193186759949, 0.30807095766067505, 0.31239989399909973, 0.3334140479564667, 0.10353845357894897, 0.32086530327796936, 0.20795077085494995, 0.671484112739563, 0.20527294278144836, 0.0, 0.2064647078514099, 0.0, 0.2842322289943695, 0.15291833877563477, 0.0, 0.0, 0.028758905827999115, 0.0, 0.0, 0.0532848984003067, 0.21567131578922272, 0.0, 0.4147211015224457, 0.2844787836074829, 0.23957182466983795, 0.30575960874557495, 0.0, 0.12454815208911896, 0.0, 0.0, 0.0, 0.10779335349798203, 0.13577388226985931, 0.08710655570030212, 0.27141597867012024, 0.0, 0.1304711252450943, 0.018196679651737213, 0.0, 0.3968903124332428, 0.31782689690589905, 0.3562062382698059, 0.3792913258075714, 0.7780655026435852, 0.32547932863235474, 0.37560850381851196, 0.33443737030029297, 0.03346162661910057, 0.0, 0.0, 0.0, 0.2826271951198578, 0.17089003324508667, 0.365955650806427, 0.2910202145576477, 0.23787929117679596, 0.4882499575614929, 0.09857811778783798, 0.32036370038986206, 0.3203338086605072, 0.024320382624864578, 0.0, 0.0, 0.08680328726768494, 0.023257602006196976, 0.0, 0.36052465438842773, 0.09846889227628708, 0.003684181021526456, 0.0, 0.0, 0.2817994952201843, 0.013196776621043682, 0.43776753544807434, 0.6481705904006958, 0.2866728603839874, 0.7450116276741028, 0.252028226852417, 0.4728197157382965, 0.22467663884162903, 0.30392277240753174, 0.03453407064080238, 0.34231412410736084, 0.14468257129192352, 0.13219209015369415, 0.0, 0.0, 0.0, 0.0, 0.08474231511354446, 0.0, 0.0, 0.02543959580361843, 0.0, 0.01711329258978367, 0.17530357837677002, 0.4746716618537903, 0.1466495543718338, 0.38687798380851746, 0.037014439702034, 0.12878474593162537, 0.05090312659740448, 0.7294142842292786, 0.4163074791431427, 0.2720993161201477, 0.42200541496276855, 0.11704161763191223, 0.264322429895401, 0.036523859947919846, 0.3325219452381134, 0.12908092141151428, 0.11271592974662781, 0.0, 0.0, 0.0, 0.2609168589115143, 0.11581955850124359, 0.2901718020439148, 0.39587897062301636, 0.12655778229236603, 0.32253575325012207, 0.006601778324693441, 0.10048890858888626, 0.15803392231464386, 0.16619952023029327, 0.1238802820444107, 0.1294466108083725, 0.3959372639656067, 0.0, 0.0, 0.18647350370883942, 0.0, 0.32489779591560364, 0.2348044365644455, 0.0781777873635292, 0.08446784317493439, 0.46471264958381653, 0.1915576308965683, 0.44487252831459045, 0.3696032762527466, 0.03622813522815704, 0.5029882192611694, 0.24568770825862885, 0.0, 0.0, 0.20797114074230194, 0.25105032324790955, 0.13308176398277283, 0.1916680485010147, 0.010050631128251553, 0.0, 0.0, 0.3194999098777771, 0.0, 0.6795520782470703, 0.6227723956108093, 0.15391601622104645, 0.04510198533535004, 0.31248125433921814, 0.0, 0.37536388635635376, 0.0, 0.005167944822460413, 0.0, 0.29216456413269043, 0.19421128928661346, 0.394413560628891, 0.29575008153915405, 0.4110984206199646, 0.41249921917915344, 0.7920240759849548, 0.3321817219257355, 0.19525690376758575, 0.10637645423412323], [0.006221972871571779, 0.27824562788009644, 0.9564735293388367, 0.19901040196418762, 0.838010847568512, 0.05611247196793556, 0.05700929835438728, 0.11986289918422699, 0.06238176301121712, 0.02357601933181286, 0.04836183041334152, 0.3502853512763977, 0.0006494870758615434, 1.1130789518356323, 0.08910931646823883, 0.018046395853161812, 0.027442462742328644, 0.005565437488257885, 0.017811525613069534, 0.0, 0.020753497257828712, 0.06211964413523674, 0.04478903487324715, 0.01491394080221653, 0.9032639861106873, 0.0005273835267871618, 0.0, 0.0, 0.0, 0.0, 0.8850080370903015, 0.07474812865257263, 0.0, 0.006777160335332155, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.00030885665910318494, 0.0, 0.0, 0.0, 0.0, 0.07364334911108017, 0.0235905684530735, 0.0037977290339767933, 0.0, 0.09752825647592545, 0.9215301275253296, 0.0, 0.0, 0.0, 0.0, 0.0, 1.1960358619689941, 0.008105989545583725, 0.15579107403755188, 0.0, 0.00012977166625205427, 0.007132899947464466, 0.0, 0.0, 0.0, 0.0, 0.05709756165742874, 0.0, 0.0, 0.00035587145248427987, 0.5392888188362122, 0.024560563266277313, 0.008304168470203876, 0.00539197726175189, 0.01916901208460331, 0.7414222359657288, 0.0, 0.0, 0.003256043652072549, 0.0, 0.13219650089740753, 0.07103464752435684, 0.0, 0.0, 0.0011673765257000923, 0.0, 0.0, 0.0, 0.0, 0.0005032396875321865, 0.4724021255970001, 0.015468482859432697, 0.11516842991113663, 0.7449396252632141, 0.0, 0.0, 0.0005239399615675211, 0.007009039167314768, 1.2188515663146973, 0.09082435816526413, 0.0, 0.0, 0.7423477172851562, 0.01105506345629692, 0.05010038614273071, 0.05322430282831192, 0.006005824543535709, 0.0074617606587708, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.029323851689696312, 0.0, 0.0, 0.0, 0.0, 0.028719449415802956, 0.009891888126730919, 0.009274360723793507, 0.1259945034980774, 0.0, 0.0, 0.0024979577865451574, 0.631228506565094, 0.0, 0.0, 0.0, 0.00947581510990858, 0.5885383486747742, 0.13083262741565704, 0.0, 0.8187709450721741, 0.0, 0.007995521649718285, 0.0, 0.7840167880058289, 0.0, 0.0, 0.0, 1.2110345363616943, 0.011829310096800327, 0.002281339606270194, 0.020430078729987144, 0.0, 0.0, 0.0, 0.13153240084648132, 0.02729417011141777, 0.007784869056195021, 0.0, 0.03381188586354256, 0.5734752416610718, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0184115432202816, 0.0, 0.006344964262098074, 0.0, 0.0, 0.0, 0.3640279173851013, 0.0, 0.0, 0.220810204744339, 0.0, 0.0, 0.0, 0.0, 0.0, 0.017721625044941902, 0.016407642513513565, 0.0, 0.0014357087202370167, 0.0, 0.0, 0.0005107074975967407, 0.008105565793812275, 0.0, 0.0, 0.017045268788933754, 0.0, 0.013508804142475128, 0.0, 0.0, 0.2829340994358063, 0.0328022837638855, 0.0029006274417042732, 0.06688777357339859, 0.0022302751895040274, 0.0, 0.0, 0.0, 0.0, 0.001157774357125163, 0.01237801369279623, 0.011402509175240993, 0.28610754013061523, 0.19723345339298248, 0.14487409591674805, 0.0, 0.023117344826459885, 0.0, 0.01199655793607235, 0.12053380161523819, 0.49761632084846497, 0.010741351172327995, 0.04345373809337616, 0.0, 0.0, 0.017426753416657448, 0.08653272688388824, 0.005919605493545532, 0.0, 0.0, 0.0, 0.10911352932453156, 0.2932019531726837, 0.07850726693868637, 0.0, 0.039654433727264404, 0.9003740549087524, 0.0, 0.0, 0.021876730024814606, 0.0, 0.028872067108750343, 0.0, 0.0, 0.17631939053535461, 1.1945277452468872, 0.01904837228357792, 0.0, 0.035939980298280716, 0.0, 0.0, 0.008884696289896965, 0.05368366092443466, 0.009792539291083813, 0.09524539858102798, 0.0, 0.0, 0.0076845912262797356, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0027275264728814363, 0.780436635017395, 0.01603499986231327, 0.010358773171901703, 0.23860704898834229, 0.15468475222587585, 1.275078296661377, 0.16829517483711243, 0.01034607831388712, 0.0, 0.0, 1.1043593883514404, 0.0, 0.00033581172465346754, 0.0, 0.0, 0.0, 0.0, 0.0413568951189518, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.009047077968716621, 0.7687130570411682, 0.014854073524475098, 0.024204222485423088, 1.2184640169143677, 0.09264176338911057, 0.012328294105827808, 0.015806956216692924, 0.0008644610643386841, 0.0, 0.0, 0.17542724311351776, 0.0, 0.047316476702690125, 0.05218953266739845, 0.018311331048607826, 0.4401741325855255, 0.0, 0.0014868308790028095, 0.0860128402709961, 0.4711141288280487, 0.06957633793354034, 0.8691742420196533, 0.0, 0.0, 0.0, 0.006625423673540354, 0.0, 0.01213905680924654, 0.8517819046974182, 0.0, 0.0, 0.06523830443620682, 0.041419826447963715, 0.007692073006182909, 0.0, 0.00128999596927315, 0.0003040047304239124, 0.816638171672821, 0.007105530705302954, 0.02646358124911785, 1.086702823638916, 0.0, 0.0, 0.757403552532196, 0.0, 0.0, 0.0, 0.07897630333900452, 0.09143079072237015, 0.0, 0.0, 0.5636830925941467, 0.0, 1.1399450302124023, 0.0686836987733841, 0.022753266617655754, 0.3186247646808624, 0.0, 0.0, 0.006726577412337065, 0.04887115955352783, 0.008814728818833828, 0.0, 0.0018791152397170663, 0.0], [0.0839681401848793, 0.2663930058479309, 0.0, 0.0, 0.13713233172893524, 0.0889037624001503, 0.07409322261810303, 0.0, 0.03606694936752319, 0.09334482252597809, 0.0, 0.18559670448303223, 0.23699717223644257, 0.0, 0.0, 0.2068232297897339, 0.2981787919998169, 0.26734182238578796, 0.0, 0.7947554588317871, 0.20988556742668152, 0.11457622796297073, 0.32389137148857117, 0.323888897895813, 0.05333505570888519, 0.4063795804977417, 0.5136920809745789, 0.23723813891410828, 0.0, 0.06794548034667969, 0.018051376566290855, 0.0, 0.18362624943256378, 0.13352814316749573, 0.01928851380944252, 0.32483842968940735, 0.0, 0.02496686950325966, 0.0, 0.17409467697143555, 0.0005123778246343136, 0.16827955842018127, 0.0, 0.4241052269935608, 0.0, 0.0, 0.032982856035232544, 0.0, 0.04776522144675255, 0.1507730782032013, 0.10927148908376694, 0.05928543582558632, 0.06444017589092255, 0.3656938374042511, 0.3055517077445984, 0.0, 0.13434617221355438, 0.0, 0.0, 0.008573412895202637, 0.0, 0.19907908141613007, 0.21409523487091064, 0.0, 0.0, 0.0, 0.22239713370800018, 0.0, 0.0, 0.15669012069702148, 0.0, 0.29449301958084106, 0.04509226232767105, 0.0, 0.05208856984972954, 0.2788238525390625, 0.0, 0.15077504515647888, 0.5222972631454468, 0.0, 0.0, 0.06399022787809372, 0.26929405331611633, 0.030016271397471428, 0.029380911961197853, 0.35430189967155457, 0.07953337579965591, 0.2592753469944, 0.0, 0.2941908538341522, 0.3683587908744812, 0.0, 0.16880905628204346, 0.0, 0.015317012555897236, 0.34343230724334717, 0.45979124307632446, 0.6928316354751587, 0.32038289308547974, 0.03027384728193283, 0.0, 0.0, 0.2555118203163147, 0.17064648866653442, 0.20772838592529297, 0.18035253882408142, 0.0, 0.0, 0.5431108474731445, 0.08846858888864517, 0.10358904302120209, 0.2146826684474945, 0.5842980146408081, 0.5300981998443604, 0.0, 0.0802341178059578, 0.18300022184848785, 0.18221323192119598, 0.25895529985427856, 0.219507098197937, 0.5098056793212891, 0.0, 0.3955797851085663, 0.0, 0.15642739832401276, 0.08885616064071655, 0.292445570230484, 0.22838862240314484, 0.7840505242347717, 0.4165317118167877, 0.17841345071792603, 0.2524975836277008, 0.6113829016685486, 0.6413407921791077, 0.44004523754119873, 0.04014794901013374, 0.0, 0.0, 0.43248221278190613, 0.12910331785678864, 0.5099201798439026, 0.15246130526065826, 0.0, 0.14097155630588531, 0.824341356754303, 0.3895772397518158, 0.0, 0.0, 0.021559972316026688, 0.11844894289970398, 0.0, 0.3788478970527649, 0.0, 0.44188329577445984, 0.0, 0.029645435512065887, 0.0, 0.0, 0.0, 0.07666661590337753, 0.0, 0.0, 0.1024143397808075, 0.24400794506072998, 0.09714743494987488, 0.05324835330247879, 0.36755436658859253, 0.3806188106536865, 0.0, 0.0, 0.0, 0.19595244526863098, 0.2946262061595917, 0.0, 0.1854817122220993, 0.0, 0.0413861982524395, 0.023397844284772873, 0.0, 0.0, 0.20683352649211884, 0.0, 0.0, 0.009100259281694889, 0.0959177166223526, 0.11629863828420639, 0.02908715046942234, 0.31288978457450867, 0.0708446279168129, 0.0, 0.10001598298549652, 0.18168097734451294, 0.5887914299964905, 0.0, 0.4755953252315521, 0.2955792248249054, 0.0052405595779418945, 0.012149146758019924, 0.0, 0.0, 0.09202095121145248, 0.1961715966463089, 0.0, 0.15108774602413177, 0.16350232064723969, 0.4328232407569885, 0.03904302790760994, 0.024014733731746674, 0.0, 0.0, 0.07143373042345047, 0.3255382776260376, 0.1271868646144867, 0.03251875936985016, 0.04320845380425453, 0.14329169690608978, 0.42893871665000916, 0.005368697922676802, 0.17273390293121338, 0.0, 0.07741953432559967, 0.0, 0.0, 0.0, 0.0, 0.02140660770237446, 0.01730484329164028, 0.11339768767356873, 0.030103124678134918, 0.012929010204970837, 0.0, 0.05008048936724663, 0.04104252904653549, 0.3639325201511383, 0.07806885242462158, 0.0, 0.26027825474739075, 0.0, 0.04662704840302467, 0.1126125305891037, 0.0, 0.0, 0.0448814257979393, 0.08072216063737869, 0.3781818151473999, 0.0, 0.0, 0.0, 0.4399786591529846, 0.0, 0.2538806200027466, 0.7225512862205505, 0.07669078558683395, 0.0, 0.5833088159561157, 0.07769273221492767, 0.26087552309036255, 0.0, 0.308814138174057, 0.0, 0.08998168259859085, 0.1806500405073166, 0.14246493577957153, 0.02587789110839367, 0.0286464411765337, 0.022808972746133804, 0.0, 0.031203757971525192, 0.07703579217195511, 0.18080943822860718, 0.1676889955997467, 0.0, 0.01774534210562706, 0.30317923426628113, 0.29432412981987, 0.0, 0.47188684344291687, 0.0, 0.008617211133241653, 0.23541806638240814, 0.38573041558265686, 0.0, 0.28252729773521423, 0.3837065100669861, 0.0, 0.1717088520526886, 0.0, 0.08314092457294464, 0.1871703714132309, 0.14906126260757446, 0.32875773310661316, 0.0, 0.0, 0.4030655324459076, 0.14377689361572266, 0.1954386830329895, 0.25857013463974, 0.13390904664993286, 0.34149014949798584, 0.0, 0.0, 0.12962846457958221, 0.23302550613880157, 0.1595286726951599, 0.005766424350440502, 0.13448145985603333, 0.0, 0.11272704601287842, 0.0, 0.14978301525115967, 0.36322319507598877, 0.0, 0.08692041784524918, 0.628768801689148, 0.2285355031490326, 0.0, 0.09437061846256256, 0.0, 0.23021918535232544, 0.40754514932632446, 0.05332975834608078, 0.0, 0.048785194754600525, 0.0, 0.14200186729431152, 0.16323894262313843, 0.18709349632263184, 0.4723016619682312, 0.0, 0.011396193876862526, 0.1415991336107254, 0.18440473079681396, 0.3518977463245392, 0.0, 0.0, 0.015782611444592476, 0.0, 0.015372063033282757, 0.05414413660764694, 0.02059822715818882, 0.0, 0.0, 0.0, 0.0802357867360115, 0.1636282205581665, 0.0, 0.040477246046066284, 0.0764855295419693, 0.010627416893839836, 0.0, 0.0, 0.4459652304649353, 0.07459867745637894], [0.011640840210020542, 0.0, 0.0, 0.06184255704283714, 0.046045150607824326, 0.0, 0.05502031370997429, 0.5856219530105591, 0.34756916761398315, 0.14772453904151917, 0.0888981819152832, 0.04027126356959343, 0.0, 0.02391444891691208, 0.398330420255661, 0.04900288209319115, 0.19831304252147675, 0.1433134824037552, 0.12696632742881775, 0.13620033860206604, 0.02860339917242527, 0.07541181892156601, 0.01184451300650835, 0.050437845289707184, 0.006227018311619759, 0.018061749637126923, 0.060546889901161194, 0.07847270369529724, 0.12378966808319092, 0.224019855260849, 0.010199504904448986, 0.0, 0.10831327736377716, 0.1584446281194687, 0.12291822582483292, 0.13410229980945587, 0.010466140694916248, 0.09798873960971832, 0.0, 0.09047955274581909, 0.23683317005634308, 0.02724391594529152, 0.08630242198705673, 0.10933838039636612, 0.6217819452285767, 0.40288445353507996, 0.23986364901065826, 0.06298797577619553, 0.01340499334037304, 0.006100803148001432, 0.0278658214956522, 0.06577538698911667, 0.03549107536673546, 0.024439165368676186, 0.07494387030601501, 0.0, 0.0, 0.0, 0.07857948541641235, 0.5793201923370361, 0.4815521240234375, 0.20163756608963013, 0.20971091091632843, 0.5006992220878601, 0.26380670070648193, 0.31115639209747314, 0.2803146541118622, 0.1862902045249939, 0.3034268617630005, 0.0763411819934845, 0.39692050218582153, 0.09269005060195923, 0.03663861006498337, 0.0892728716135025, 0.0, 0.0, 0.052756596356630325, 0.0017949865432456136, 0.0, 0.0, 0.2533290386199951, 0.08596258610486984, 0.03892742097377777, 0.0, 0.0, 0.0, 0.0, 0.005206396337598562, 0.026368483901023865, 0.0, 0.0132749667391181, 0.03110593371093273, 0.05904645100235939, 0.7126699090003967, 0.16637298464775085, 0.09954996407032013, 0.14979536831378937, 0.23209701478481293, 0.040362145751714706, 0.03607887774705887, 0.0, 0.7168018817901611, 0.22218798100948334, 0.18249370157718658, 0.0, 0.044376492500305176, 0.31057116389274597, 0.684370219707489, 0.06004071608185768, 0.2876310348510742, 0.0, 0.207382470369339, 0.11551111936569214, 0.15665991604328156, 0.025405602529644966, 0.0, 0.20950953662395477, 0.10347333550453186, 0.037942830473184586, 0.12197371572256088, 0.006028409581631422, 0.03296256065368652, 0.053103793412446976, 0.35720402002334595, 0.0, 0.0, 0.04591243341565132, 0.0, 0.0, 0.16304874420166016, 0.0667654499411583, 0.0, 0.06833050400018692, 0.0, 0.0, 0.015452059917151928, 0.0, 0.21116547286510468, 0.2615181803703308, 0.0, 0.010891949757933617, 0.13076625764369965, 0.02185252122581005, 0.0, 0.045943837612867355, 0.13918974995613098, 0.0, 0.07878777384757996, 0.7676336765289307, 0.30740898847579956, 0.5165329575538635, 0.09166716039180756, 0.0077486359514296055, 0.03557705879211426, 0.0, 0.05758585035800934, 0.37032344937324524, 0.010920356027781963, 0.2044529765844345, 0.007598698604851961, 0.0, 0.0, 0.0, 0.18666896224021912, 0.0, 0.3211297392845154, 0.16124017536640167, 0.012667723000049591, 0.0, 0.0010857480810955167, 0.0, 0.056292127817869186, 0.21859805285930634, 0.3524583876132965, 0.09892388433218002, 0.2548050284385681, 0.22847692668437958, 0.0, 0.0, 0.03053894266486168, 0.04257163405418396, 0.0, 0.00025416960124857724, 0.0, 0.0, 0.0, 0.0710717961192131, 0.0016024798387661576, 0.18302984535694122, 0.47583508491516113, 0.03667399287223816, 0.06647557020187378, 0.06176834926009178, 0.0, 0.12135394662618637, 0.004610169213265181, 0.7862015962600708, 0.525719940662384, 0.4458221197128296, 0.5140130519866943, 0.2701072692871094, 0.11587578058242798, 0.23243293166160583, 0.0, 0.0, 0.0, 0.018277565017342567, 0.06575541943311691, 0.0, 0.23789365589618683, 0.2649063467979431, 0.009759841486811638, 0.006531108170747757, 0.03595665842294693, 0.21695351600646973, 0.03906004875898361, 0.08935326337814331, 0.3196120262145996, 0.1250380426645279, 0.2609381675720215, 0.02958354353904724, 0.0, 0.004482578486204147, 0.0, 0.00048513786168769, 0.03838353976607323, 0.23605605959892273, 0.0, 0.2886641323566437, 0.03658827021718025, 0.2599026560783386, 0.001389703364111483, 0.0, 0.0, 0.0, 0.27971652150154114, 0.0, 0.010638176463544369, 0.048619840294122696, 0.08752128481864929, 0.05157700181007385, 0.7587136030197144, 0.47250863909721375, 0.2605597972869873, 0.03860597684979439, 0.021695224568247795, 0.2670460045337677, 0.660881519317627, 0.08432340621948242, 0.2882356345653534, 0.06635114550590515, 0.09935746341943741, 0.00521894171833992, 0.0, 0.0, 0.0, 0.010583395138382912, 0.0130091467872262, 0.04662894830107689, 0.3052097260951996, 0.15257632732391357, 0.0, 0.005260061472654343, 0.015370577573776245, 0.07629702985286713, 0.1940530687570572, 0.04452565684914589, 0.34515905380249023, 0.3885015845298767, 0.18463118374347687, 0.03375965356826782, 0.11370307952165604, 0.796366274356842, 0.010239665396511555, 0.09475432336330414, 0.3283473253250122, 0.07330120354890823, 0.0818348079919815, 0.6699621081352234, 0.026278814300894737, 0.17583265900611877, 0.0, 0.016103550791740417, 0.04210425913333893, 0.0, 0.14794063568115234, 0.3142278492450714, 0.15363425016403198, 0.0, 0.0038336387369781733, 0.11412905156612396, 0.007779378909617662, 0.7244942784309387, 0.0015822761924937367, 0.0969170480966568, 0.08047118037939072, 0.05549396201968193, 0.0049771517515182495, 0.01286323368549347, 0.0013405086938291788, 0.21720312535762787, 0.0215893667191267, 0.07941987365484238, 0.05329779163002968, 0.6583001613616943, 0.07678637653589249, 0.0, 0.0, 0.28161805868148804, 0.0, 0.019845088943839073, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.2628318667411804, 0.07727647572755814, 0.1299903392791748, 0.02209140732884407, 0.03745169937610626, 0.4603034257888794, 0.26121586561203003, 0.0, 0.0, 0.046672843396663666, 0.0, 0.0718572586774826, 0.7800763845443726, 0.08837835490703583, 0.0037825009785592556, 0.0, 0.0, 0.0, 0.0, 0.09797287732362747, 0.0858379378914833, 0.06155204027891159, 0.03269592672586441, 0.0, 0.03425084426999092, 0.3766329288482666, 0.10101913660764694, 0.11510857939720154, 0.24401874840259552, 0.2287788838148117, 0.05808525159955025, 0.0, 0.0, 0.0, 0.07991086691617966, 0.006936749443411827]]]},\n",
       "            {\n",
       "            'hltrCFG': {'tokenization_config': {\"token_prefix\": \"\\u2581\", \"partial_token_prefix\": \"\"}\n",
       "                }\n",
       "            })\n",
       "         }, function (err) {\n",
       "            console.log(err);\n",
       "        })"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "text = ''' Now I ask you: \\n what can be expected of man since he is a being endowed with strange qualities? Shower upon him every earthly blessing, drown him in a sea of happiness, so that nothing but bubbles of bliss can be seen on the surface; give him economic prosperity, such that he should have nothing else to do but sleep, eat cakes and busy himself with the continuation of his species, and even then out of sheer ingratitude, sheer spite, man would play you some nasty trick. He would even risk his cakes and would deliberately desire the most fatal rubbish, the most uneconomical absurdity, simply to introduce into all this positive good sense his fatal fantastic element. It is just his fantastic dreams, his vulgar folly that he will desire to retain, simply in order to prove to himself--as though that were so necessary-- that men still are men and not the keys of a piano, which the laws of nature threaten to control so completely that soon one will be able to desire nothing but by the calendar. And that is not all: even if man really were nothing but a piano-key, even if this were proved to him by natural science and mathematics, even then he would not become reasonable, but would purposely do something perverse out of simple ingratitude, simply to gain his point. And if he does not find means he will contrive destruction and chaos, will contrive sufferings of all sorts, only to gain his point! He will launch a curse upon the world, and as only man can curse (it is his privilege, the primary distinction between him and other animals), may be by his curse alone he will attain his object--that is, convince himself that he is a man and not a piano-key!\n",
    "'''\n",
    "\n",
    "inputs = ecco_model.tokenizer([text], return_tensors=\"pt\")\n",
    "output = ecco_model(inputs)\n",
    "\n",
    "nmf_1 = output.run_nmf(n_components=8) \n",
    "\n",
    "nmf_1.explore()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1734dc15",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'embedding': 'embeddings.word_embeddings',\n",
       " 'type': 'mlm',\n",
       " 'activations': ['intermediate\\\\.dense'],\n",
       " 'token_prefix': '▁',\n",
       " 'partial_token_prefix': '',\n",
       " 'tokenizer_config': {'token_prefix': '▁', 'partial_token_prefix': ''}}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output.config"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "101dd50f",
   "metadata": {},
   "source": [
    "## model-config.yaml\n",
    "If we're adding to model-config.yaml, the entry looks like this:\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "00250ac0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n\\nmicrosoft/deberta-v3-small:\\n    embedding: \"model.embeddings.word_embeddings\"\\n    type: \\'mlm\\'\\n    activations:\\n        - \\'intermediate\\\\.dense\\'\\n    token_prefix: \\'▁\\'\\n    partial_token_prefix: \\'\\'\\n\\n'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "\n",
    "microsoft/deberta-v3-small:\n",
    "    embedding: \"embeddings.word_embeddings\"\n",
    "    type: 'mlm'\n",
    "    activations:\n",
    "        - 'intermediate\\.dense'\n",
    "    token_prefix: '▁'\n",
    "    partial_token_prefix: ''\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05b45ba3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}