TextSummarization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ноутбук создан на основе семинара Гусева Ильи на кафедре компьютерной лингвистики МФТИ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!wget -q https://www.dropbox.com/s/43l702z5a5i2w8j/gazeta_train.txt\n",
    "!wget -q https://www.dropbox.com/s/k2egt3sug0hb185/gazeta_val.txt\n",
    "!wget -q https://www.dropbox.com/s/3gki5n5djs9w0v6/gazeta_test.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "SXS1sdYZCluU"
   },
   "outputs": [],
   "source": [
    "!pip install --upgrade razdel allennlp torch fasttext OpenNMT-py networkx pymorphy2 nltk rouge==0.3.1 summa\n",
    "!pip install transformers youtokentome catalyst\n",
    "!pip install flair"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import BertTokenizer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from transformers import AutoModelWithLMHead\n",
    "from nltk.translate.bleu_score import corpus_bleu\n",
    "from rouge import Rouge\n",
    "import torch\n",
    "import random\n",
    "from scipy.spatial import distance\n",
    "import numpy as np\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.nn.utils.rnn import pack_padded_sequence as pack\n",
    "from torch.nn.utils.rnn import pad_packed_sequence as unpack\n",
    "\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5pZ2UGS2DGjH"
   },
   "outputs": [],
   "source": [
    "# Function for reading text files\n",
    "\n",
    "def read_gazeta_records(file_name, shuffle=True, sort_by_date=False):\n",
    "    assert shuffle != sort_by_date\n",
    "    records = []\n",
    "    with open(file_name, \"r\") as r:\n",
    "        for line in r:\n",
    "            records.append(eval(line)) # Simple hack\n",
    "    records = pd.DataFrame(records)\n",
    "    if sort_by_date:\n",
    "        records = records.sort(\"date\")\n",
    "    if shuffle:\n",
    "        records = records.sample(frac=1)\n",
    "    return records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GNDp-BunEA91"
   },
   "outputs": [],
   "source": [
    "# Read text files: train, validation, test\n",
    "\n",
    "train_records = read_gazeta_records(\"gazeta_train.txt\")\n",
    "val_records = read_gazeta_records(\"gazeta_val.txt\")\n",
    "test_records = read_gazeta_records(\"gazeta_test.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VQn3XSV0EcU-"
   },
   "outputs": [],
   "source": [
    "# Function to calculate scores: Bleu & Rouge\n",
    "\n",
    "def calc_scores(references, predictions, metric=\"all\"):\n",
    "    print(\"Count:\", len(predictions))\n",
    "    print(\"Ref:\", references[-1])\n",
    "    print(\"Hyp:\", predictions[-1])\n",
    "\n",
    "    if metric in (\"bleu\", \"all\"):\n",
    "        print(\"BLEU: \", corpus_bleu([[r] for r in references], predictions))\n",
    "    if metric in (\"rouge\", \"all\"):\n",
    "        rouge = Rouge()\n",
    "        scores = rouge.get_scores(predictions, references, avg=True)\n",
    "        print(\"ROUGE: \", scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "QsAcVSli3r3S"
   },
   "source": [
    "### TextRank method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "eOVGcPUH-YI8"
   },
   "outputs": [],
   "source": [
    "# Load FastText sentence embeddings trained on wiki ru, it is a mean over word embeddings\n",
    "\n",
    "from flair.embeddings import WordEmbeddings, BytePairEmbeddings, StackedEmbeddings\n",
    "from flair.data import Sentence\n",
    "from flair.embeddings import DocumentPoolEmbeddings\n",
    "\n",
    "pooled_embeddings = DocumentPoolEmbeddings([WordEmbeddings('ru')], pooling='mean')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "g_k9QBeM643-"
   },
   "outputs": [],
   "source": [
    "# Load DeepPavlov Bert embeddings and tokenizer\n",
    "\n",
    "tokenizer_bert = BertTokenizer.from_pretrained(\"DeepPavlov/rubert-base-cased\")\n",
    "model_bert = AutoModelWithLMHead.from_pretrained(\"DeepPavlov/rubert-base-cased\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "rs36gqEb7ZbW"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 124
    },
    "colab_type": "code",
    "id": "m2GwyRrMPAzS",
    "outputId": "e5c14038-3011-4ce0-9e63-c779322a7969"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Count: 1\n",
      "Ref: евросоюз тянет с санкциями против турции, поэтому кипр заблокировал введение ограничительных мер против нескольких чиновников из крыма, сообщает reuters со ссылкой на три дипломатических источника. никосия отрицает, что связывает эти два вопроса, и говорит, что ей просто нужно время для анализа предложенных ограничительных мер против россиян.\n",
      "Hyp: это тем более удивительно, поскольку кипр пользовался беспрецедентной поддержкой и солидарностью со своими партнерами по ес в течение последних нескольких месяцев — в споре с турцией», — сказал один из дипломатов ес. в июне 2019 года тогдашний председатель еврокомиссии жан-клод юнкер заявил, что действия турецкой стороны в иэз кипра неприемлемы, поэтому анкара будет наказана жесткими санкциями.\n",
      "BLEU:  0.351127959006569\n",
      "ROUGE:  {'rouge-1': {'f': 0.19565216897211735, 'p': 0.17647058823529413, 'r': 0.21951219512195122}, 'rouge-2': {'f': 0.0, 'p': 0.0, 'r': 0.0}, 'rouge-l': {'f': 0.12745817871463636, 'p': 0.11764705882352941, 'r': 0.14634146341463414}}\n"
     ]
    }
   ],
   "source": [
    "from itertools import combinations\n",
    "import networkx as nx\n",
    "import numpy as np\n",
    "import torch\n",
    "import razdel\n",
    "import pymorphy2\n",
    "\n",
    "\n",
    " \n",
    "def unique_words_similarity(words1, words2):\n",
    "    '''\n",
    "    Function to calculate sentences similarity based on words intersection\n",
    "    ''' \n",
    "    words1 = set(words1)\n",
    "    words2 = set(words2)\n",
    "    if not len(words1) or not len(words2):\n",
    "        return 0.0\n",
    "    return len(words1.intersection(words2))/(np.log10(len(words1)) + np.log10(len(words2)))\n",
    "\n",
    "def similarity_FastTextFlair(words1, words2):\n",
    "    '''\n",
    "    Function to calculate sentences similarity based on cosine distance between FastText embeddings (Flair)\n",
    "    ''' \n",
    "    sentence1 = Sentence(' '.join(words1))\n",
    "    sentence2 = Sentence(' '.join(words2))\n",
    "\n",
    "    # embed with pooled embeddings\n",
    "    with torch.no_grad():\n",
    "        pooled_embeddings.embed(sentence1)\n",
    "        pooled_embeddings.embed(sentence2)\n",
    "\n",
    "    vec1 = sentence1.embedding.cpu()\n",
    "    vec2 = sentence2.embedding.cpu()\n",
    "    return distance.cosine(vec1, vec2) # np.dot(vec1, vec2) / (np.linalg.norm(vec1) * np.linalg.norm(vec2))\n",
    "\n",
    "def similarity_FastText(words1, words2):\n",
    "    '''\n",
    "    Function to calculate sentences similarity based on cosine distance between FastText embeddings (ft_native_300_ru_wiki_lenta_lower_case)\n",
    "    ''' \n",
    "    sentence1 = ' '.join(words1)\n",
    "    sentence2 = ' '.join(words2)\n",
    "\n",
    "    # embed with pooled embeddings\n",
    "    vec1 = model_ft.get_sentence_vector(sentence1)\n",
    "    vec2 = model_ft.get_sentence_vector(sentence2)\n",
    "\n",
    "    # To CPU\n",
    "    # vec1 = vec1.cpu()\n",
    "    # vec2 = vec2.cpu()\n",
    "    return np.dot(vec1, vec2) / (np.linalg.norm(vec1) * np.linalg.norm(vec2))\n",
    "\n",
    "def similarity_DeepPavlovBert(words1,  words2):\n",
    "    '''\n",
    "    Function to calculate sentences similarity based on cosine distance between DeepPavlov embeddings\n",
    "    ''' \n",
    "    input_ids1, input_ids2 = torch.tensor([tokenizer_bert.encode(words1)]), torch.tensor([tokenizer_bert.encode(words2)])\n",
    "    with torch.no_grad():\n",
    "        output_tuple1, output_tuple2 = model_bert(input_ids1), model_bert(input_ids2)\n",
    "        last_hidden_states1, last_hidden_states2 = output_tuple1[0], output_tuple2[0]\n",
    "\n",
    "\n",
    "    cos = torch.nn.CosineSimilarity(1)\n",
    "    # print(last_hidden_states2.mean(1).shape)\n",
    "    dist_s = cos(last_hidden_states2.mean(1), last_hidden_states1.mean(1))\n",
    " \n",
    "    # dist_s = scipy.spatial.distance.cdist([emb1], emb2, \"cosine\")[0]\n",
    "    return dist_s.item()\n",
    "    \n",
    "\n",
    "def gen_text_rank_summary(text, calc_similarity=unique_words_similarity, summary_part=0.1, lower=True, morph=None):\n",
    "    '''\n",
    "    Summary generation using TextRank\n",
    "    '''\n",
    "    # Split text in sentences\n",
    "    sentences = [sentence.text for sentence in razdel.sentenize(text)]\n",
    "    n_sentences = len(sentences)\n",
    "\n",
    "    # Tokenize sentences\n",
    "    sentences_words = [[token.text.lower() if lower else token.text for token in razdel.tokenize(sentence)] for sentence in sentences]\n",
    "\n",
    "    # Lemmatize words if necessary \n",
    "    if morph is not None:\n",
    "        sentences_words = [[morph.parse(word)[0].normal_form for word in words] for words in sentences_words]\n",
    "\n",
    "    # For each sentences pair calculate similarity\n",
    "    pairs = combinations(range(n_sentences), 2)\n",
    "    scores = [(i, j, calc_similarity(sentences_words[i], sentences_words[j])) for i, j in pairs]\n",
    "    \n",
    "\n",
    "    # Build a graph with edges that are equal similarity value between sentences\n",
    "    g = nx.Graph()\n",
    "    g.add_weighted_edges_from(scores)\n",
    "\n",
    "    # Calculate PageRank\n",
    "    pr = nx.pagerank(g)\n",
    "    result = [(i, pr[i], s) for i, s in enumerate(sentences) if i in pr]\n",
    "    result.sort(key=lambda x: x[1], reverse=True)\n",
    "\n",
    "    # Choose top sentences\n",
    "    n_summary_sentences = max(int(n_sentences * summary_part), 1)\n",
    "    result = result[:n_summary_sentences]\n",
    "\n",
    "    # Restore their original order\n",
    "    result.sort(key=lambda x: x[0])\n",
    "\n",
    "    # Restore summary text\n",
    "    predicted_summary = \" \".join([sentence for i, proba, sentence in result])\n",
    "    predicted_summary = predicted_summary.lower() if lower else predicted_summary\n",
    "    return predicted_summary\n",
    "\n",
    "def calc_text_rank_score(records, calc_similarity=unique_words_similarity, summary_part=0.1, lower=True, nrows=1000, morph=None):\n",
    "    '''\n",
    "    Function to calculate TextRank score\n",
    "    '''\n",
    "    references = []\n",
    "    predictions = []\n",
    "\n",
    "    for text, summary in records[['text', 'summary']].values[:nrows]:\n",
    "        summary = summary if not lower else summary.lower()\n",
    "        references.append(summary)\n",
    "\n",
    "        predicted_summary = gen_text_rank_summary(text, calc_similarity, summary_part, lower, morph=morph)\n",
    "        text = text if not lower else text.lower()\n",
    "        predictions.append(predicted_summary)\n",
    "\n",
    "    calc_scores(references, predictions)\n",
    "\n",
    "# calc_text_rank_score(test_records, calc_similarity=unique_words_similarity)\n",
    "calc_text_rank_score(test_records.iloc[:1], calc_similarity=similarity_DeepPavlovBert)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "xdTrfxycB7cd"
   },
   "source": [
    "### Extractive RNN "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 208
    },
    "colab_type": "code",
    "id": "Bdwnc7vHGIxM",
    "outputId": "81235ca2-a756-4e9d-ed1b-c2690f61a55f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2020-05-25 08:25:10--  http://files.deeppavlov.ai/embeddings/ft_native_300_ru_wiki_lenta_lower_case/ft_native_300_ru_wiki_lenta_lower_case.bin\n",
      "Resolving files.deeppavlov.ai (files.deeppavlov.ai)... 93.175.29.74\n",
      "Connecting to files.deeppavlov.ai (files.deeppavlov.ai)|93.175.29.74|:80... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 6216266514 (5.8G) [application/octet-stream]\n",
      "Saving to: ‘ft_native_300_ru_wiki_lenta_lower_case.bin’\n",
      "\n",
      "ft_native_300_ru_wi 100%[===================>]   5.79G  2.45MB/s    in 74m 45s \n",
      "\n",
      "2020-05-25 09:39:56 (1.32 MB/s) - ‘ft_native_300_ru_wiki_lenta_lower_case.bin’ saved [6216266514/6216266514]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Load DeepPavlov FastText embeddings\n",
    "!wget http://files.deeppavlov.ai/embeddings/ft_native_300_ru_wiki_lenta_lower_case/ft_native_300_ru_wiki_lenta_lower_case.bin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "13Qjj5uLGQFF",
    "outputId": "4bee1775-9382-49c6-abfd-8c3a45025f45"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n"
     ]
    }
   ],
   "source": [
    "# Load bin file\n",
    "model_ft = fasttext.load_model('ft_native_300_ru_wiki_lenta_lower_case.bin')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Sxsc0Orf8hGq"
   },
   "outputs": [],
   "source": [
    "import copy\n",
    "import razdel\n",
    "import random\n",
    "\n",
    "def build_oracle_summary_greedy(text, gold_summary, calc_score, lower=True, max_sentences=30):\n",
    "    '''\n",
    "    Жадное построение oracle summary\n",
    "    '''\n",
    "    gold_summary = gold_summary.lower() if lower else gold_summary\n",
    "    # Split text in sentences\n",
    "    sentences = [sentence.text.lower() if lower else sentence.text for sentence in razdel.sentenize(text)][:max_sentences]\n",
    "    n_sentences = len(sentences)\n",
    "    oracle_summary_sentences = set()\n",
    "    score = -1.0\n",
    "    summaries = []\n",
    "    for _ in range(min(n_sentences, 2)):\n",
    "        for i in range(n_sentences):\n",
    "            if i in oracle_summary_sentences:\n",
    "                continue\n",
    "            current_summary_sentences = copy.copy(oracle_summary_sentences)\n",
    "            # Add some sentences to the existing summary\n",
    "            current_summary_sentences.add(i)\n",
    "            current_summary = \" \".join([sentences[index] for index in sorted(list(current_summary_sentences))])\n",
    "            # Count metrics\n",
    "            current_score = calc_score(current_summary, gold_summary)\n",
    "            summaries.append((current_score, current_summary_sentences))\n",
    "        # If metrics have improved with addition of a sentence, then try to add more\n",
    "        # Else break\n",
    "        best_summary_score, best_summary_sentences = max(summaries)\n",
    "        if best_summary_score <= score:\n",
    "            break\n",
    "        oracle_summary_sentences = best_summary_sentences\n",
    "        score = best_summary_score\n",
    "    oracle_summary = \" \".join([sentences[index] for index in sorted(list(oracle_summary_sentences))])\n",
    "    return oracle_summary, oracle_summary_sentences\n",
    "\n",
    "def calc_single_score(pred_summary, gold_summary, rouge):\n",
    "    return rouge.get_scores([pred_summary], [gold_summary], avg=True)['rouge-2']['f']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 225,
     "referenced_widgets": [
      "e44d5ff38cd944e5b5b4c9d225881562",
      "2e566e55ab0e4687af556fff5f990bb4",
      "d10a80939c6c49b5b6fbe8103e69c1fc",
      "a4a1e862c16343aababbf404d6b56a5c",
      "c08fbdfa99e54d15bb45254d23d53b1d",
      "01bc56666c214eb4ad497074b3c94ab9",
      "53c1a290165941c2b247885c5f8082bb",
      "105b402e7ece4df79cae6b720a80d210"
     ]
    },
    "colab_type": "code",
    "id": "7T_ak-KDB8rp",
    "outputId": "6ca48923-a4eb-4750-f524-a409e70a266a"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:8: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
      "  \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e44d5ff38cd944e5b5b4c9d225881562",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1000.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Count: 1000\n",
      "Ref: глава роскосмоса дмитрий рогозин раскритиковал шутку ивана урганта об упрощении требований к кандидатам в космонавты. гендиректор госкорпорации заявил об утрате ведущим чувства юмора после восьми лет непрерывной работы на шоу «вечерний ургант».\n",
      "Hyp: генеральный директор роскосмоса дмитрий рогозин прокомментировал шутку телеведущего ивана урганта об упрощении подготовки космонавтов. ранее глава госкорпорации заявил о переговорах с центром подготовки космонавтов об отказе от некоторых требований к кандидатам.\n",
      "BLEU:  0.43106535769975873\n",
      "ROUGE:  {'rouge-1': {'f': 0.3522902373664272, 'p': 0.4307700039374759, 'r': 0.31660276824452355}, 'rouge-2': {'f': 0.2032850093782878, 'p': 0.25546554153070994, 'r': 0.18111381667522672}, 'rouge-l': {'f': 0.2997182027550175, 'p': 0.39783483642945483, 'r': 0.2913698482908416}}\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm_notebook as tqdm\n",
    "\n",
    "# Calculate score of oracle summarization\n",
    "\n",
    "def calc_oracle_score(records, nrows=1000, lower=True):\n",
    "    references = []\n",
    "    predictions = []\n",
    "    rouge = Rouge()\n",
    "  \n",
    "    for text, summary in tqdm(records[['text', 'summary']].values[:nrows]):\n",
    "        summary = summary if not lower else summary.lower()\n",
    "        references.append(summary)\n",
    "        predicted_summary, _ = build_oracle_summary_greedy(text, summary, calc_score=lambda x, y: calc_single_score(x, y, rouge))\n",
    "        predictions.append(predicted_summary)\n",
    "\n",
    "    calc_scores(references, predictions)\n",
    "\n",
    "calc_oracle_score(test_records)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 217,
     "referenced_widgets": [
      "e97deeb0dbd34763918da3cd1ce2d96d",
      "0fd4eec972554953b90aedb4b00c5bca",
      "3a6749d40e1044828bc8f92095ae0c9d",
      "0732f432e8c34c3dbada9b2d4b48c6a0",
      "81470fa4151f47d78102e74378ab79c1",
      "22ff06420fca4347bd4ef0d8285a596d",
      "92a7a3bdfc0e470985a12802c51fd31d",
      "4e413af1bc2b46d6ae78cdfee9a73c89",
      "c181c1cbad4c457d933f64048c305ce8",
      "b3f3a55a4ef142019faa85f0e110b017",
      "3e8cadbac2194b90b649ecce8b3651c3",
      "2494a76c1e484baa9c50214dae08f6c6",
      "62eaace7e0f44762a2c97be3b896d94e",
      "db070b630e554e49abeed94367cdb882",
      "39321aefe22844b0bf3f5b0306e344bd",
      "ecb74cb487064e1cb3f7e1451886b1b3",
      "05f3590569264fb8ab3165fd71c6c22c",
      "8e4f8f6e414f4f57be5f2bdab68b05a0",
      "b2555242f7ff4c758c4f50b5292ec2bb",
      "f7f228f323e44697b234ac01850f68d0",
      "a3a0111965bd4fd3852c9b096c875001",
      "7b7306aa9b044908a13cffd959287717",
      "11a778c6b88643df9f542240e99c84a4",
      "8e91ebac38b4493288c3cfd28c0959e4"
     ]
    },
    "colab_type": "code",
    "id": "Jdb-39jO-72q",
    "outputId": "340a3946-0fc8-4135-a496-bc7625d27dd3"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:10: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
      "  # Remove the CWD from sys.path while we load stuff.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e97deeb0dbd34763918da3cd1ce2d96d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=4096.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c181c1cbad4c457d933f64048c305ce8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=256.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "05f3590569264fb8ab3165fd71c6c22c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=256.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from rouge import Rouge\n",
    "\n",
    "def add_oracle_summary_to_records(records, max_sentences=30, lower=True, nrows=1000):\n",
    "    rouge = Rouge()\n",
    "    sentences_ = []\n",
    "    oracle_sentences_ = []\n",
    "    oracle_summary_ = []\n",
    "    records = records.iloc[:nrows].copy()\n",
    "\n",
    "    for text, summary in tqdm(records[['text', 'summary']].values):\n",
    "        summary = summary.lower() if lower else summary\n",
    "        sentences = [sentence.text.lower() if lower else sentence.text for sentence in razdel.sentenize(text)][:max_sentences]\n",
    "        oracle_summary, sentences_indicies = build_oracle_summary_greedy(text, summary, calc_score=lambda x, y: calc_single_score(x, y, rouge),\n",
    "                                                                         lower=lower, max_sentences=max_sentences)\n",
    "        sentences_ += [sentences]\n",
    "        oracle_sentences_ += [list(sentences_indicies)]\n",
    "        oracle_summary_ += [oracle_summary]\n",
    "    records['sentences'] = sentences_\n",
    "    records['oracle_sentences'] = oracle_sentences_\n",
    "    records['oracle_summary'] = oracle_summary_\n",
    "    return records\n",
    "\n",
    "ext_train_records = add_oracle_summary_to_records(train_records, nrows=4096)\n",
    "ext_val_records = add_oracle_summary_to_records(val_records, nrows=256)\n",
    "ext_test_records = add_oracle_summary_to_records(test_records, nrows=256)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MNyxstTChK3C"
   },
   "outputs": [],
   "source": [
    "import random\n",
    "import math\n",
    "import torch\n",
    "import numpy as np\n",
    "from rouge import Rouge\n",
    "\n",
    "\n",
    "from torch.utils import data\n",
    "\n",
    "# Dataset generating \n",
    "\n",
    "class ExtDataset(data.Dataset):\n",
    "    def __init__(self, records, lower=True, max_sentences=30, max_sentence_length=50, device=torch.device('cpu')):\n",
    "        self.records = records\n",
    "        self.num_samples = records.shape[0]\n",
    "        self.lower = lower\n",
    "        self.rouge = Rouge()\n",
    "        self.max_sentences = max_sentences\n",
    "        self.max_sentence_length = max_sentence_length\n",
    "        self.device = device\n",
    "        \n",
    "    def __len__(self):\n",
    "        return self.records.shape[0]\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        cur_record = self.records.iloc[idx]\n",
    "        inputs = list()\n",
    "        # 1: DeepPavlov\n",
    "        for sent in cur_record['sentences']:\n",
    "            emb_sent = torch.tensor(model_e.get_sentence_vector(sent))\n",
    "            inputs.append(emb_sent.cpu())\n",
    "        # 2: Flair\n",
    "        # for sent in cur_record['sentences']:\n",
    "        #     sentence = Sentence(sent)\n",
    "        #     with torch.no_grad():\n",
    "        #         pooled_embeddings.embed(sentence)\n",
    "\n",
    "            # inputs.append(sentence.embedding.cpu())\n",
    "        # inputs = list(map(lambda x: x[:self.max_sentence_length], self.bpe_processor.encode(cur_record['sentences'], output_type=yttm.OutputType.ID)))\n",
    "        outputs = [int(i in cur_record['oracle_sentences']) for i in range(len(cur_record['sentences']))]\n",
    "        return {'inputs': inputs, 'outputs': outputs}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "JegA9fOMsZN5"
   },
   "outputs": [],
   "source": [
    "train_dataset = ExtDataset(ext_train_records)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bvARjudojEDD"
   },
   "outputs": [],
   "source": [
    "# Function to generate batches\n",
    "def collate_fn(records):\n",
    "    max_length = max(len(sentence) for record in records for sentence in record['inputs'])\n",
    "    max_sentences = max(len(record['outputs']) for record in records)\n",
    "\n",
    "    new_inputs = torch.zeros((len(records), max_sentences, max_length))\n",
    "    new_outputs = torch.zeros((len(records), max_sentences))\n",
    "    for i, record in enumerate(records):\n",
    "        for j, sentence in enumerate(record['inputs']):\n",
    "            new_inputs[i, j, :len(sentence)] += np.array(sentence)\n",
    "        new_outputs[i, :len(record['outputs'])] += np.array(record['outputs'])\n",
    "    return {'features': new_inputs.type(torch.LongTensor), 'targets': new_outputs}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 107
    },
    "colab_type": "code",
    "id": "aWlf7XdheJUN",
    "outputId": "916332b5-a244-4dc8-d1ec-7c8a56b51d34"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/rnn.py:50: UserWarning:\n",
      "\n",
      "dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.2 and num_layers=1\n",
      "\n"
     ]
    }
   ],
   "source": [
    "class SentenceTaggerRNN(nn.Module):\n",
    "    def __init__(self,\n",
    "                 token_embedding_dim=300,\n",
    "                 sentence_encoder_hidden_size=300,\n",
    "                 hidden_size=300,\n",
    "                 bidirectional=True,\n",
    "                 sentence_encoder_n_layers=2,\n",
    "                 sentence_encoder_dropout=0.2,\n",
    "                 sentence_encoder_bidirectional=True,\n",
    "                 n_layers=1,\n",
    "                 dropout=0.2):\n",
    "        super(SentenceTaggerRNN, self).__init__()\n",
    "\n",
    "        num_directions = 2 if bidirectional else 1\n",
    "        assert hidden_size % num_directions == 0\n",
    "        hidden_size = hidden_size // num_directions\n",
    "\n",
    "        self.hidden_size = hidden_size\n",
    "        self.n_layers = n_layers\n",
    "        self.dropout = dropout\n",
    "        self.bidirectional = bidirectional\n",
    "\n",
    "        \n",
    "        self.rnn_layer = nn.LSTM(sentence_encoder_hidden_size, hidden_size, n_layers, dropout=dropout,\n",
    "                           bidirectional=bidirectional, batch_first=True)\n",
    "        self.dropout_layer = nn.Dropout(dropout)\n",
    "        self.content_linear_layer = nn.Linear(hidden_size * 2, 1)\n",
    "        self.document_linear_layer = nn.Linear(hidden_size * 2, hidden_size * 2)\n",
    "        self.salience_linear_layer = nn.Linear(hidden_size * 2, hidden_size * 2)\n",
    "        self.tanh_layer = nn.Tanh()\n",
    "\n",
    "    def forward(self, inputs, hidden=None):\n",
    "        batch_size = inputs.size(0)\n",
    "        sentences_count = inputs.size(1)\n",
    "        tokens_count = inputs.size(2)\n",
    "        inputs = inputs.reshape(-1, tokens_count)\n",
    "        embedded_sentences = inputs\n",
    "        embedded_sentences = embedded_sentences.reshape(batch_size, sentences_count, -1)\n",
    "        outputs, _ = self.rnn_layer(embedded_sentences.float(), hidden)\n",
    "        outputs = self.dropout_layer(outputs)\n",
    "        document_embedding = self.tanh_layer(self.document_linear_layer(torch.mean(outputs, 1)))\n",
    "        content = self.content_linear_layer(outputs).squeeze(2)\n",
    "        salience = torch.bmm(outputs, self.salience_linear_layer(document_embedding).unsqueeze(2)).squeeze(2)\n",
    "        m = torch.nn.Softmax()\n",
    "        return m(content + salience)\n",
    "\n",
    "model = SentenceTaggerRNN()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4q2Gb6ODHHB_"
   },
   "source": [
    "### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "UVDW8raJeQxn",
    "outputId": "fbaec370-87f0-4c83-ee73-bc6f497b7bf8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):   0% 0/32 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):   0% 0/32 [00:02<?, ?it/s, loss=0.708]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):   3% 1/32 [00:02<01:25,  2.76s/it, loss=0.708]\u001b[A\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:84: UserWarning:\n",
      "\n",
      "Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):   3% 1/32 [00:05<01:25,  2.76s/it, loss=0.708]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):   6% 2/32 [00:05<01:20,  2.70s/it, loss=0.708]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):   6% 2/32 [00:07<01:20,  2.70s/it, loss=0.708]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):   9% 3/32 [00:07<01:16,  2.64s/it, loss=0.708]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):   9% 3/32 [00:10<01:16,  2.64s/it, loss=0.708]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  12% 4/32 [00:10<01:12,  2.60s/it, loss=0.708]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  12% 4/32 [00:12<01:12,  2.60s/it, loss=0.708]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  16% 5/32 [00:12<01:09,  2.58s/it, loss=0.708]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  16% 5/32 [00:15<01:09,  2.58s/it, loss=0.708]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  19% 6/32 [00:15<01:06,  2.54s/it, loss=0.708]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  19% 6/32 [00:17<01:06,  2.54s/it, loss=0.707]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  22% 7/32 [00:17<01:03,  2.56s/it, loss=0.707]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  22% 7/32 [00:20<01:03,  2.56s/it, loss=0.707]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  25% 8/32 [00:20<01:01,  2.56s/it, loss=0.707]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  25% 8/32 [00:22<01:01,  2.56s/it, loss=0.706]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  28% 9/32 [00:22<00:58,  2.54s/it, loss=0.706]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  28% 9/32 [00:25<00:58,  2.54s/it, loss=0.705]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  31% 10/32 [00:25<00:56,  2.56s/it, loss=0.705]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  31% 10/32 [00:28<00:56,  2.56s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  34% 11/32 [00:28<00:53,  2.57s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  34% 11/32 [00:30<00:53,  2.57s/it, loss=0.701]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  38% 12/32 [00:30<00:51,  2.56s/it, loss=0.701]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  38% 12/32 [00:33<00:51,  2.56s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  41% 13/32 [00:33<00:48,  2.55s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  41% 13/32 [00:35<00:48,  2.55s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  44% 14/32 [00:35<00:45,  2.54s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  44% 14/32 [00:38<00:45,  2.54s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  47% 15/32 [00:38<00:43,  2.54s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  47% 15/32 [00:40<00:43,  2.54s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  50% 16/32 [00:40<00:40,  2.53s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  50% 16/32 [00:43<00:40,  2.53s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  53% 17/32 [00:43<00:38,  2.55s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  53% 17/32 [00:46<00:38,  2.55s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  56% 18/32 [00:46<00:36,  2.58s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  56% 18/32 [00:48<00:36,  2.58s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  59% 19/32 [00:48<00:33,  2.56s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  59% 19/32 [00:51<00:33,  2.56s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  62% 20/32 [00:51<00:30,  2.58s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  62% 20/32 [00:53<00:30,  2.58s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  66% 21/32 [00:53<00:28,  2.58s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  66% 21/32 [00:56<00:28,  2.58s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  69% 22/32 [00:56<00:25,  2.55s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  69% 22/32 [00:58<00:25,  2.55s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  72% 23/32 [00:58<00:22,  2.53s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  72% 23/32 [01:01<00:22,  2.53s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  75% 24/32 [01:01<00:20,  2.54s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  75% 24/32 [01:03<00:20,  2.54s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  78% 25/32 [01:03<00:17,  2.51s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  78% 25/32 [01:06<00:17,  2.51s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  81% 26/32 [01:06<00:15,  2.52s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  81% 26/32 [01:08<00:15,  2.52s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  84% 27/32 [01:08<00:12,  2.49s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  84% 27/32 [01:11<00:12,  2.49s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  88% 28/32 [01:11<00:09,  2.49s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  88% 28/32 [01:13<00:09,  2.49s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  91% 29/32 [01:13<00:07,  2.49s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  91% 29/32 [01:16<00:07,  2.49s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  94% 30/32 [01:16<00:04,  2.47s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  94% 30/32 [01:18<00:04,  2.47s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  97% 31/32 [01:18<00:02,  2.48s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train):  97% 31/32 [01:21<00:02,  2.48s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train): 100% 32/32 [01:21<00:00,  2.47s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (train): 100% 32/32 [01:21<00:00,  2.53s/it, loss=0.697]\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (valid):   0% 0/2 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (valid):   0% 0/2 [00:02<?, ?it/s, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (valid):  50% 1/2 [00:02<00:02,  2.52s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (valid):  50% 1/2 [00:05<00:02,  2.52s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (valid): 100% 2/2 [00:05<00:00,  2.53s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (valid): 100% 2/2 [00:05<00:00,  2.55s/it, loss=0.695]\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (test):   0% 0/2 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (test):   0% 0/2 [00:02<?, ?it/s, loss=0.694]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (test):  50% 1/2 [00:02<00:02,  2.49s/it, loss=0.694]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (test):  50% 1/2 [00:04<00:02,  2.49s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (test): 100% 2/2 [00:04<00:00,  2.49s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "1/7 * Epoch (test): 100% 2/2 [00:05<00:00,  2.50s/it, loss=0.696]\n",
      "[2020-05-25 10:07:07,482] \n",
      "1/7 * Epoch 1 (_base): lr=0.0010 | momentum=0.9000\n",
      "1/7 * Epoch 1 (train): loss=0.7008\n",
      "1/7 * Epoch 1 (valid): loss=0.6951\n",
      "1/7 * Epoch 1 (test): loss=0.6945\n",
      "[2020-05-25 10:07:07,482] \n",
      "1/7 * Epoch 1 (_base): lr=0.0010 | momentum=0.9000\n",
      "1/7 * Epoch 1 (train): loss=0.7008\n",
      "1/7 * Epoch 1 (valid): loss=0.6951\n",
      "1/7 * Epoch 1 (test): loss=0.6945\n",
      "[2020-05-25 10:07:07,482] \n",
      "1/7 * Epoch 1 (_base): lr=0.0010 | momentum=0.9000\n",
      "1/7 * Epoch 1 (train): loss=0.7008\n",
      "1/7 * Epoch 1 (valid): loss=0.6951\n",
      "1/7 * Epoch 1 (test): loss=0.6945\n",
      "[2020-05-25 10:07:07,482] \n",
      "1/7 * Epoch 1 (_base): lr=0.0010 | momentum=0.9000\n",
      "1/7 * Epoch 1 (train): loss=0.7008\n",
      "1/7 * Epoch 1 (valid): loss=0.6951\n",
      "1/7 * Epoch 1 (test): loss=0.6945\n",
      "[2020-05-25 10:07:07,482] \n",
      "1/7 * Epoch 1 (_base): lr=0.0010 | momentum=0.9000\n",
      "1/7 * Epoch 1 (train): loss=0.7008\n",
      "1/7 * Epoch 1 (valid): loss=0.6951\n",
      "1/7 * Epoch 1 (test): loss=0.6945\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):   0% 0/32 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):   0% 0/32 [00:02<?, ?it/s, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):   3% 1/32 [00:02<01:17,  2.52s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):   3% 1/32 [00:04<01:17,  2.52s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):   6% 2/32 [00:04<01:14,  2.48s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):   6% 2/32 [00:07<01:14,  2.48s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):   9% 3/32 [00:07<01:11,  2.48s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):   9% 3/32 [00:09<01:11,  2.48s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  12% 4/32 [00:09<01:09,  2.48s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  12% 4/32 [00:12<01:09,  2.48s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  16% 5/32 [00:12<01:07,  2.51s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  16% 5/32 [00:14<01:07,  2.51s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  19% 6/32 [00:14<01:05,  2.51s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  19% 6/32 [00:17<01:05,  2.51s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  22% 7/32 [00:17<01:02,  2.52s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  22% 7/32 [00:20<01:02,  2.52s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  25% 8/32 [00:20<01:00,  2.52s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  25% 8/32 [00:22<01:00,  2.52s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  28% 9/32 [00:22<00:58,  2.55s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  28% 9/32 [00:25<00:58,  2.55s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  31% 10/32 [00:25<00:55,  2.53s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  31% 10/32 [00:27<00:55,  2.53s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  34% 11/32 [00:27<00:53,  2.55s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  34% 11/32 [00:30<00:53,  2.55s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  38% 12/32 [00:30<00:51,  2.57s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  38% 12/32 [00:32<00:51,  2.57s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  41% 13/32 [00:32<00:49,  2.59s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  41% 13/32 [00:35<00:49,  2.59s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  44% 14/32 [00:35<00:46,  2.59s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  44% 14/32 [00:38<00:46,  2.59s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  47% 15/32 [00:38<00:43,  2.57s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  47% 15/32 [00:40<00:43,  2.57s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  50% 16/32 [00:40<00:40,  2.55s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  50% 16/32 [00:43<00:40,  2.55s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  53% 17/32 [00:43<00:38,  2.55s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  53% 17/32 [00:45<00:38,  2.55s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  56% 18/32 [00:45<00:35,  2.53s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  56% 18/32 [00:48<00:35,  2.53s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  59% 19/32 [00:48<00:32,  2.52s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  59% 19/32 [00:50<00:32,  2.52s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  62% 20/32 [00:50<00:30,  2.51s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  62% 20/32 [00:53<00:30,  2.51s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  66% 21/32 [00:53<00:27,  2.51s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  66% 21/32 [00:55<00:27,  2.51s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  69% 22/32 [00:55<00:24,  2.50s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  69% 22/32 [00:58<00:24,  2.50s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  72% 23/32 [00:58<00:22,  2.53s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  72% 23/32 [01:00<00:22,  2.53s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  75% 24/32 [01:00<00:20,  2.54s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  75% 24/32 [01:03<00:20,  2.54s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  78% 25/32 [01:03<00:18,  2.59s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  78% 25/32 [01:06<00:18,  2.59s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  81% 26/32 [01:06<00:15,  2.58s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  81% 26/32 [01:08<00:15,  2.58s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  84% 27/32 [01:08<00:12,  2.57s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  84% 27/32 [01:11<00:12,  2.57s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  88% 28/32 [01:11<00:10,  2.57s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  88% 28/32 [01:13<00:10,  2.57s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  91% 29/32 [01:13<00:07,  2.57s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  91% 29/32 [01:16<00:07,  2.57s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  94% 30/32 [01:16<00:05,  2.56s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  94% 30/32 [01:18<00:05,  2.56s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  97% 31/32 [01:18<00:02,  2.57s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train):  97% 31/32 [01:21<00:02,  2.57s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train): 100% 32/32 [01:21<00:00,  2.56s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (train): 100% 32/32 [01:21<00:00,  2.54s/it, loss=0.697]\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (valid):   0% 0/2 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (valid):   0% 0/2 [00:02<?, ?it/s, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (valid):  50% 1/2 [00:02<00:02,  2.62s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (valid):  50% 1/2 [00:05<00:02,  2.62s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (valid): 100% 2/2 [00:05<00:00,  2.62s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (valid): 100% 2/2 [00:05<00:00,  2.62s/it, loss=0.695]\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (test):   0% 0/2 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (test):   0% 0/2 [00:02<?, ?it/s, loss=0.694]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (test):  50% 1/2 [00:02<00:02,  2.60s/it, loss=0.694]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (test):  50% 1/2 [00:05<00:02,  2.60s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (test): 100% 2/2 [00:05<00:00,  2.59s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "2/7 * Epoch (test): 100% 2/2 [00:05<00:00,  2.58s/it, loss=0.696]\n",
      "[2020-05-25 10:08:39,341] \n",
      "2/7 * Epoch 2 (_base): lr=0.0010 | momentum=0.9000\n",
      "2/7 * Epoch 2 (train): loss=0.6977\n",
      "2/7 * Epoch 2 (valid): loss=0.6951\n",
      "2/7 * Epoch 2 (test): loss=0.6945\n",
      "[2020-05-25 10:08:39,341] \n",
      "2/7 * Epoch 2 (_base): lr=0.0010 | momentum=0.9000\n",
      "2/7 * Epoch 2 (train): loss=0.6977\n",
      "2/7 * Epoch 2 (valid): loss=0.6951\n",
      "2/7 * Epoch 2 (test): loss=0.6945\n",
      "[2020-05-25 10:08:39,341] \n",
      "2/7 * Epoch 2 (_base): lr=0.0010 | momentum=0.9000\n",
      "2/7 * Epoch 2 (train): loss=0.6977\n",
      "2/7 * Epoch 2 (valid): loss=0.6951\n",
      "2/7 * Epoch 2 (test): loss=0.6945\n",
      "[2020-05-25 10:08:39,341] \n",
      "2/7 * Epoch 2 (_base): lr=0.0010 | momentum=0.9000\n",
      "2/7 * Epoch 2 (train): loss=0.6977\n",
      "2/7 * Epoch 2 (valid): loss=0.6951\n",
      "2/7 * Epoch 2 (test): loss=0.6945\n",
      "[2020-05-25 10:08:39,341] \n",
      "2/7 * Epoch 2 (_base): lr=0.0010 | momentum=0.9000\n",
      "2/7 * Epoch 2 (train): loss=0.6977\n",
      "2/7 * Epoch 2 (valid): loss=0.6951\n",
      "2/7 * Epoch 2 (test): loss=0.6945\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):   0% 0/32 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):   0% 0/32 [00:02<?, ?it/s, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):   3% 1/32 [00:02<01:17,  2.51s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):   3% 1/32 [00:05<01:17,  2.51s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):   6% 2/32 [00:05<01:16,  2.56s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):   6% 2/32 [00:07<01:16,  2.56s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):   9% 3/32 [00:07<01:14,  2.56s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):   9% 3/32 [00:10<01:14,  2.56s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  12% 4/32 [00:10<01:11,  2.57s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  12% 4/32 [00:12<01:11,  2.57s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  16% 5/32 [00:12<01:09,  2.58s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  16% 5/32 [00:15<01:09,  2.58s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  19% 6/32 [00:15<01:07,  2.58s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  19% 6/32 [00:18<01:07,  2.58s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  22% 7/32 [00:18<01:05,  2.61s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  22% 7/32 [00:20<01:05,  2.61s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  25% 8/32 [00:20<01:02,  2.60s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  25% 8/32 [00:23<01:02,  2.60s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  28% 9/32 [00:23<00:59,  2.60s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  28% 9/32 [00:25<00:59,  2.60s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  31% 10/32 [00:25<00:57,  2.60s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  31% 10/32 [00:28<00:57,  2.60s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  34% 11/32 [00:28<00:54,  2.59s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  34% 11/32 [00:31<00:54,  2.59s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  38% 12/32 [00:31<00:53,  2.65s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  38% 12/32 [00:33<00:53,  2.65s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  41% 13/32 [00:33<00:49,  2.62s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  41% 13/32 [00:36<00:49,  2.62s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  44% 14/32 [00:36<00:47,  2.63s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  44% 14/32 [00:39<00:47,  2.63s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  47% 15/32 [00:39<00:44,  2.61s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  47% 15/32 [00:41<00:44,  2.61s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  50% 16/32 [00:41<00:41,  2.60s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  50% 16/32 [00:44<00:41,  2.60s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  53% 17/32 [00:44<00:39,  2.60s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  53% 17/32 [00:46<00:39,  2.60s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  56% 18/32 [00:46<00:36,  2.59s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  56% 18/32 [00:49<00:36,  2.59s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  59% 19/32 [00:49<00:33,  2.57s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  59% 19/32 [00:51<00:33,  2.57s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  62% 20/32 [00:51<00:30,  2.56s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  62% 20/32 [00:54<00:30,  2.56s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  66% 21/32 [00:54<00:28,  2.55s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  66% 21/32 [00:57<00:28,  2.55s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  69% 22/32 [00:57<00:25,  2.56s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  69% 22/32 [00:59<00:25,  2.56s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  72% 23/32 [00:59<00:23,  2.60s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  72% 23/32 [01:02<00:23,  2.60s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  75% 24/32 [01:02<00:20,  2.59s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  75% 24/32 [01:04<00:20,  2.59s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  78% 25/32 [01:04<00:18,  2.59s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  78% 25/32 [01:07<00:18,  2.59s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  81% 26/32 [01:07<00:15,  2.58s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  81% 26/32 [01:09<00:15,  2.58s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  84% 27/32 [01:10<00:12,  2.57s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  84% 27/32 [01:12<00:12,  2.57s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  88% 28/32 [01:12<00:10,  2.58s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  88% 28/32 [01:15<00:10,  2.58s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  91% 29/32 [01:15<00:07,  2.57s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  91% 29/32 [01:17<00:07,  2.57s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  94% 30/32 [01:17<00:05,  2.54s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  94% 30/32 [01:20<00:05,  2.54s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  97% 31/32 [01:20<00:02,  2.55s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train):  97% 31/32 [01:22<00:02,  2.55s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train): 100% 32/32 [01:22<00:00,  2.55s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (train): 100% 32/32 [01:22<00:00,  2.59s/it, loss=0.697]\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (valid):   0% 0/2 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (valid):   0% 0/2 [00:02<?, ?it/s, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (valid):  50% 1/2 [00:02<00:02,  2.72s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (valid):  50% 1/2 [00:05<00:02,  2.72s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (valid): 100% 2/2 [00:05<00:00,  2.68s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (valid): 100% 2/2 [00:05<00:00,  2.66s/it, loss=0.695]\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (test):   0% 0/2 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (test):   0% 0/2 [00:02<?, ?it/s, loss=0.694]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (test):  50% 1/2 [00:02<00:02,  2.74s/it, loss=0.694]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (test):  50% 1/2 [00:05<00:02,  2.74s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (test): 100% 2/2 [00:05<00:00,  2.78s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "3/7 * Epoch (test): 100% 2/2 [00:05<00:00,  2.81s/it, loss=0.696]\n",
      "[2020-05-25 10:10:13,090] \n",
      "3/7 * Epoch 3 (_base): lr=0.0010 | momentum=0.9000\n",
      "3/7 * Epoch 3 (train): loss=0.6977\n",
      "3/7 * Epoch 3 (valid): loss=0.6951\n",
      "3/7 * Epoch 3 (test): loss=0.6945\n",
      "[2020-05-25 10:10:13,090] \n",
      "3/7 * Epoch 3 (_base): lr=0.0010 | momentum=0.9000\n",
      "3/7 * Epoch 3 (train): loss=0.6977\n",
      "3/7 * Epoch 3 (valid): loss=0.6951\n",
      "3/7 * Epoch 3 (test): loss=0.6945\n",
      "[2020-05-25 10:10:13,090] \n",
      "3/7 * Epoch 3 (_base): lr=0.0010 | momentum=0.9000\n",
      "3/7 * Epoch 3 (train): loss=0.6977\n",
      "3/7 * Epoch 3 (valid): loss=0.6951\n",
      "3/7 * Epoch 3 (test): loss=0.6945\n",
      "[2020-05-25 10:10:13,090] \n",
      "3/7 * Epoch 3 (_base): lr=0.0010 | momentum=0.9000\n",
      "3/7 * Epoch 3 (train): loss=0.6977\n",
      "3/7 * Epoch 3 (valid): loss=0.6951\n",
      "3/7 * Epoch 3 (test): loss=0.6945\n",
      "[2020-05-25 10:10:13,090] \n",
      "3/7 * Epoch 3 (_base): lr=0.0010 | momentum=0.9000\n",
      "3/7 * Epoch 3 (train): loss=0.6977\n",
      "3/7 * Epoch 3 (valid): loss=0.6951\n",
      "3/7 * Epoch 3 (test): loss=0.6945\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):   0% 0/32 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):   0% 0/32 [00:02<?, ?it/s, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):   3% 1/32 [00:02<01:26,  2.79s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):   3% 1/32 [00:05<01:26,  2.79s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):   6% 2/32 [00:05<01:21,  2.70s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):   6% 2/32 [00:07<01:21,  2.70s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):   9% 3/32 [00:07<01:16,  2.64s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):   9% 3/32 [00:10<01:16,  2.64s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  12% 4/32 [00:10<01:13,  2.62s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  12% 4/32 [00:12<01:13,  2.62s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  16% 5/32 [00:12<01:10,  2.60s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  16% 5/32 [00:15<01:10,  2.60s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  19% 6/32 [00:15<01:07,  2.59s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  19% 6/32 [00:17<01:07,  2.59s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  22% 7/32 [00:17<01:03,  2.56s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  22% 7/32 [00:20<01:03,  2.56s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  25% 8/32 [00:20<01:02,  2.59s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  25% 8/32 [00:23<01:02,  2.59s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  28% 9/32 [00:23<00:59,  2.59s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  28% 9/32 [00:25<00:59,  2.59s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  31% 10/32 [00:25<00:56,  2.58s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  31% 10/32 [00:28<00:56,  2.58s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  34% 11/32 [00:28<00:54,  2.60s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  34% 11/32 [00:30<00:54,  2.60s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  38% 12/32 [00:30<00:51,  2.59s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  38% 12/32 [00:33<00:51,  2.59s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  41% 13/32 [00:33<00:49,  2.61s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  41% 13/32 [00:36<00:49,  2.61s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  44% 14/32 [00:36<00:46,  2.60s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  44% 14/32 [00:38<00:46,  2.60s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  47% 15/32 [00:38<00:44,  2.59s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  47% 15/32 [00:41<00:44,  2.59s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  50% 16/32 [00:41<00:41,  2.62s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  50% 16/32 [00:44<00:41,  2.62s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  53% 17/32 [00:44<00:39,  2.62s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  53% 17/32 [00:46<00:39,  2.62s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  56% 18/32 [00:46<00:36,  2.62s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  56% 18/32 [00:49<00:36,  2.62s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  59% 19/32 [00:49<00:33,  2.59s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  59% 19/32 [00:51<00:33,  2.59s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  62% 20/32 [00:51<00:30,  2.58s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  62% 20/32 [00:54<00:30,  2.58s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  66% 21/32 [00:54<00:28,  2.59s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  66% 21/32 [00:56<00:28,  2.59s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  69% 22/32 [00:56<00:25,  2.58s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  69% 22/32 [00:59<00:25,  2.58s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  72% 23/32 [00:59<00:23,  2.61s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  72% 23/32 [01:02<00:23,  2.61s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  75% 24/32 [01:02<00:20,  2.61s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  75% 24/32 [01:04<00:20,  2.61s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  78% 25/32 [01:04<00:18,  2.61s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  78% 25/32 [01:07<00:18,  2.61s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  81% 26/32 [01:07<00:15,  2.60s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  81% 26/32 [01:09<00:15,  2.60s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  84% 27/32 [01:09<00:12,  2.59s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  84% 27/32 [01:12<00:12,  2.59s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  88% 28/32 [01:12<00:10,  2.58s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  88% 28/32 [01:15<00:10,  2.58s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  91% 29/32 [01:15<00:07,  2.56s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  91% 29/32 [01:17<00:07,  2.56s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  94% 30/32 [01:17<00:05,  2.56s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  94% 30/32 [01:20<00:05,  2.56s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  97% 31/32 [01:20<00:02,  2.58s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train):  97% 31/32 [01:22<00:02,  2.58s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train): 100% 32/32 [01:22<00:00,  2.60s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (train): 100% 32/32 [01:22<00:00,  2.59s/it, loss=0.697]\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (valid):   0% 0/2 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (valid):   0% 0/2 [00:02<?, ?it/s, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (valid):  50% 1/2 [00:02<00:02,  2.59s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (valid):  50% 1/2 [00:05<00:02,  2.59s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (valid): 100% 2/2 [00:05<00:00,  2.60s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (valid): 100% 2/2 [00:05<00:00,  2.62s/it, loss=0.695]\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (test):   0% 0/2 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (test):   0% 0/2 [00:02<?, ?it/s, loss=0.694]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (test):  50% 1/2 [00:02<00:02,  2.59s/it, loss=0.694]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (test):  50% 1/2 [00:05<00:02,  2.59s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (test): 100% 2/2 [00:05<00:00,  2.64s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "4/7 * Epoch (test): 100% 2/2 [00:05<00:00,  2.68s/it, loss=0.696]\n",
      "[2020-05-25 10:11:46,657] \n",
      "4/7 * Epoch 4 (_base): lr=0.0010 | momentum=0.9000\n",
      "4/7 * Epoch 4 (train): loss=0.6977\n",
      "4/7 * Epoch 4 (valid): loss=0.6951\n",
      "4/7 * Epoch 4 (test): loss=0.6945\n",
      "[2020-05-25 10:11:46,657] \n",
      "4/7 * Epoch 4 (_base): lr=0.0010 | momentum=0.9000\n",
      "4/7 * Epoch 4 (train): loss=0.6977\n",
      "4/7 * Epoch 4 (valid): loss=0.6951\n",
      "4/7 * Epoch 4 (test): loss=0.6945\n",
      "[2020-05-25 10:11:46,657] \n",
      "4/7 * Epoch 4 (_base): lr=0.0010 | momentum=0.9000\n",
      "4/7 * Epoch 4 (train): loss=0.6977\n",
      "4/7 * Epoch 4 (valid): loss=0.6951\n",
      "4/7 * Epoch 4 (test): loss=0.6945\n",
      "[2020-05-25 10:11:46,657] \n",
      "4/7 * Epoch 4 (_base): lr=0.0010 | momentum=0.9000\n",
      "4/7 * Epoch 4 (train): loss=0.6977\n",
      "4/7 * Epoch 4 (valid): loss=0.6951\n",
      "4/7 * Epoch 4 (test): loss=0.6945\n",
      "[2020-05-25 10:11:46,657] \n",
      "4/7 * Epoch 4 (_base): lr=0.0010 | momentum=0.9000\n",
      "4/7 * Epoch 4 (train): loss=0.6977\n",
      "4/7 * Epoch 4 (valid): loss=0.6951\n",
      "4/7 * Epoch 4 (test): loss=0.6945\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):   0% 0/32 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):   0% 0/32 [00:02<?, ?it/s, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):   3% 1/32 [00:02<01:23,  2.70s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):   3% 1/32 [00:05<01:23,  2.70s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):   6% 2/32 [00:05<01:19,  2.65s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):   6% 2/32 [00:07<01:19,  2.65s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):   9% 3/32 [00:07<01:16,  2.63s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):   9% 3/32 [00:10<01:16,  2.63s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  12% 4/32 [00:10<01:12,  2.60s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  12% 4/32 [00:12<01:12,  2.60s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  16% 5/32 [00:12<01:10,  2.60s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  16% 5/32 [00:15<01:10,  2.60s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  19% 6/32 [00:15<01:07,  2.58s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  19% 6/32 [00:18<01:07,  2.58s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  22% 7/32 [00:18<01:04,  2.59s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  22% 7/32 [00:20<01:04,  2.59s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  25% 8/32 [00:20<01:02,  2.60s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  25% 8/32 [00:23<01:02,  2.60s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  28% 9/32 [00:23<00:59,  2.58s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  28% 9/32 [00:25<00:59,  2.58s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  31% 10/32 [00:25<00:57,  2.60s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  31% 10/32 [00:28<00:57,  2.60s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  34% 11/32 [00:28<00:54,  2.61s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  34% 11/32 [00:31<00:54,  2.61s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  38% 12/32 [00:31<00:52,  2.64s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  38% 12/32 [00:33<00:52,  2.64s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  41% 13/32 [00:33<00:49,  2.63s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  41% 13/32 [00:36<00:49,  2.63s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  44% 14/32 [00:36<00:47,  2.63s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  44% 14/32 [00:39<00:47,  2.63s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  47% 15/32 [00:39<00:44,  2.60s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  47% 15/32 [00:41<00:44,  2.60s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  50% 16/32 [00:41<00:41,  2.59s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  50% 16/32 [00:44<00:41,  2.59s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  53% 17/32 [00:44<00:38,  2.57s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  53% 17/32 [00:46<00:38,  2.57s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  56% 18/32 [00:46<00:35,  2.56s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  56% 18/32 [00:49<00:35,  2.56s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  59% 19/32 [00:49<00:32,  2.54s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  59% 19/32 [00:51<00:32,  2.54s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  62% 20/32 [00:51<00:30,  2.54s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  62% 20/32 [00:54<00:30,  2.54s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  66% 21/32 [00:54<00:27,  2.53s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  66% 21/32 [00:56<00:27,  2.53s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  69% 22/32 [00:56<00:25,  2.52s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  69% 22/32 [00:59<00:25,  2.52s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  72% 23/32 [00:59<00:22,  2.53s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  72% 23/32 [01:01<00:22,  2.53s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  75% 24/32 [01:01<00:20,  2.60s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  75% 24/32 [01:04<00:20,  2.60s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  78% 25/32 [01:04<00:18,  2.64s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  78% 25/32 [01:07<00:18,  2.64s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  81% 26/32 [01:07<00:15,  2.62s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  81% 26/32 [01:09<00:15,  2.62s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  84% 27/32 [01:09<00:12,  2.58s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  84% 27/32 [01:12<00:12,  2.58s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  88% 28/32 [01:12<00:10,  2.58s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  88% 28/32 [01:14<00:10,  2.58s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  91% 29/32 [01:14<00:07,  2.56s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  91% 29/32 [01:17<00:07,  2.56s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  94% 30/32 [01:17<00:05,  2.54s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  94% 30/32 [01:19<00:05,  2.54s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  97% 31/32 [01:19<00:02,  2.55s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train):  97% 31/32 [01:22<00:02,  2.55s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train): 100% 32/32 [01:22<00:00,  2.55s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (train): 100% 32/32 [01:22<00:00,  2.58s/it, loss=0.697]\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (valid):   0% 0/2 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (valid):   0% 0/2 [00:02<?, ?it/s, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (valid):  50% 1/2 [00:02<00:02,  2.58s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (valid):  50% 1/2 [00:05<00:02,  2.58s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (valid): 100% 2/2 [00:05<00:00,  2.60s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (valid): 100% 2/2 [00:05<00:00,  2.61s/it, loss=0.695]\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (test):   0% 0/2 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (test):   0% 0/2 [00:02<?, ?it/s, loss=0.694]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (test):  50% 1/2 [00:02<00:02,  2.61s/it, loss=0.694]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (test):  50% 1/2 [00:05<00:02,  2.61s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (test): 100% 2/2 [00:05<00:00,  2.62s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "5/7 * Epoch (test): 100% 2/2 [00:05<00:00,  2.63s/it, loss=0.696]\n",
      "[2020-05-25 10:13:19,712] \n",
      "5/7 * Epoch 5 (_base): lr=0.0010 | momentum=0.9000\n",
      "5/7 * Epoch 5 (train): loss=0.6977\n",
      "5/7 * Epoch 5 (valid): loss=0.6951\n",
      "5/7 * Epoch 5 (test): loss=0.6945\n",
      "[2020-05-25 10:13:19,712] \n",
      "5/7 * Epoch 5 (_base): lr=0.0010 | momentum=0.9000\n",
      "5/7 * Epoch 5 (train): loss=0.6977\n",
      "5/7 * Epoch 5 (valid): loss=0.6951\n",
      "5/7 * Epoch 5 (test): loss=0.6945\n",
      "[2020-05-25 10:13:19,712] \n",
      "5/7 * Epoch 5 (_base): lr=0.0010 | momentum=0.9000\n",
      "5/7 * Epoch 5 (train): loss=0.6977\n",
      "5/7 * Epoch 5 (valid): loss=0.6951\n",
      "5/7 * Epoch 5 (test): loss=0.6945\n",
      "[2020-05-25 10:13:19,712] \n",
      "5/7 * Epoch 5 (_base): lr=0.0010 | momentum=0.9000\n",
      "5/7 * Epoch 5 (train): loss=0.6977\n",
      "5/7 * Epoch 5 (valid): loss=0.6951\n",
      "5/7 * Epoch 5 (test): loss=0.6945\n",
      "[2020-05-25 10:13:19,712] \n",
      "5/7 * Epoch 5 (_base): lr=0.0010 | momentum=0.9000\n",
      "5/7 * Epoch 5 (train): loss=0.6977\n",
      "5/7 * Epoch 5 (valid): loss=0.6951\n",
      "5/7 * Epoch 5 (test): loss=0.6945\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):   0% 0/32 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):   0% 0/32 [00:02<?, ?it/s, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):   3% 1/32 [00:02<01:23,  2.69s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):   3% 1/32 [00:05<01:23,  2.69s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):   6% 2/32 [00:05<01:18,  2.63s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):   6% 2/32 [00:07<01:18,  2.63s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):   9% 3/32 [00:07<01:16,  2.62s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):   9% 3/32 [00:10<01:16,  2.62s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  12% 4/32 [00:10<01:13,  2.62s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  12% 4/32 [00:12<01:13,  2.62s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  16% 5/32 [00:12<01:10,  2.60s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  16% 5/32 [00:15<01:10,  2.60s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  19% 6/32 [00:15<01:06,  2.57s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  19% 6/32 [00:17<01:06,  2.57s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  22% 7/32 [00:17<01:03,  2.55s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  22% 7/32 [00:20<01:03,  2.55s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  25% 8/32 [00:20<01:01,  2.55s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  25% 8/32 [00:23<01:01,  2.55s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  28% 9/32 [00:23<00:58,  2.54s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  28% 9/32 [00:25<00:58,  2.54s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  31% 10/32 [00:25<00:56,  2.57s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  31% 10/32 [00:28<00:56,  2.57s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  34% 11/32 [00:28<00:54,  2.60s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  34% 11/32 [00:30<00:54,  2.60s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  38% 12/32 [00:30<00:52,  2.61s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  38% 12/32 [00:33<00:52,  2.61s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  41% 13/32 [00:33<00:49,  2.63s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  41% 13/32 [00:36<00:49,  2.63s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  44% 14/32 [00:36<00:47,  2.63s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  44% 14/32 [00:38<00:47,  2.63s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  47% 15/32 [00:38<00:44,  2.62s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  47% 15/32 [00:41<00:44,  2.62s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  50% 16/32 [00:41<00:41,  2.61s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  50% 16/32 [00:44<00:41,  2.61s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  53% 17/32 [00:44<00:39,  2.65s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  53% 17/32 [00:46<00:39,  2.65s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  56% 18/32 [00:46<00:36,  2.62s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  56% 18/32 [00:49<00:36,  2.62s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  59% 19/32 [00:49<00:33,  2.60s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  59% 19/32 [00:51<00:33,  2.60s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  62% 20/32 [00:51<00:31,  2.62s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  62% 20/32 [00:54<00:31,  2.62s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  66% 21/32 [00:54<00:28,  2.59s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  66% 21/32 [00:57<00:28,  2.59s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  69% 22/32 [00:57<00:25,  2.59s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  69% 22/32 [00:59<00:25,  2.59s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  72% 23/32 [00:59<00:23,  2.59s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  72% 23/32 [01:02<00:23,  2.59s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  75% 24/32 [01:02<00:20,  2.62s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  75% 24/32 [01:04<00:20,  2.62s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  78% 25/32 [01:04<00:18,  2.60s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  78% 25/32 [01:07<00:18,  2.60s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  81% 26/32 [01:07<00:15,  2.60s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  81% 26/32 [01:10<00:15,  2.60s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  84% 27/32 [01:10<00:13,  2.61s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  84% 27/32 [01:12<00:13,  2.61s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  88% 28/32 [01:12<00:10,  2.59s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  88% 28/32 [01:15<00:10,  2.59s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  91% 29/32 [01:15<00:07,  2.57s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  91% 29/32 [01:17<00:07,  2.57s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  94% 30/32 [01:17<00:05,  2.54s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  94% 30/32 [01:20<00:05,  2.54s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  97% 31/32 [01:20<00:02,  2.55s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train):  97% 31/32 [01:22<00:02,  2.55s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train): 100% 32/32 [01:22<00:00,  2.53s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (train): 100% 32/32 [01:22<00:00,  2.59s/it, loss=0.697]\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (valid):   0% 0/2 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (valid):   0% 0/2 [00:02<?, ?it/s, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (valid):  50% 1/2 [00:02<00:02,  2.69s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (valid):  50% 1/2 [00:05<00:02,  2.69s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (valid): 100% 2/2 [00:05<00:00,  2.68s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (valid): 100% 2/2 [00:05<00:00,  2.69s/it, loss=0.695]\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (test):   0% 0/2 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (test):   0% 0/2 [00:02<?, ?it/s, loss=0.694]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (test):  50% 1/2 [00:02<00:02,  2.61s/it, loss=0.694]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (test):  50% 1/2 [00:05<00:02,  2.61s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (test): 100% 2/2 [00:05<00:00,  2.62s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "6/7 * Epoch (test): 100% 2/2 [00:05<00:00,  2.64s/it, loss=0.696]\n",
      "[2020-05-25 10:14:53,208] \n",
      "6/7 * Epoch 6 (_base): lr=0.0010 | momentum=0.9000\n",
      "6/7 * Epoch 6 (train): loss=0.6977\n",
      "6/7 * Epoch 6 (valid): loss=0.6951\n",
      "6/7 * Epoch 6 (test): loss=0.6945\n",
      "[2020-05-25 10:14:53,208] \n",
      "6/7 * Epoch 6 (_base): lr=0.0010 | momentum=0.9000\n",
      "6/7 * Epoch 6 (train): loss=0.6977\n",
      "6/7 * Epoch 6 (valid): loss=0.6951\n",
      "6/7 * Epoch 6 (test): loss=0.6945\n",
      "[2020-05-25 10:14:53,208] \n",
      "6/7 * Epoch 6 (_base): lr=0.0010 | momentum=0.9000\n",
      "6/7 * Epoch 6 (train): loss=0.6977\n",
      "6/7 * Epoch 6 (valid): loss=0.6951\n",
      "6/7 * Epoch 6 (test): loss=0.6945\n",
      "[2020-05-25 10:14:53,208] \n",
      "6/7 * Epoch 6 (_base): lr=0.0010 | momentum=0.9000\n",
      "6/7 * Epoch 6 (train): loss=0.6977\n",
      "6/7 * Epoch 6 (valid): loss=0.6951\n",
      "6/7 * Epoch 6 (test): loss=0.6945\n",
      "[2020-05-25 10:14:53,208] \n",
      "6/7 * Epoch 6 (_base): lr=0.0010 | momentum=0.9000\n",
      "6/7 * Epoch 6 (train): loss=0.6977\n",
      "6/7 * Epoch 6 (valid): loss=0.6951\n",
      "6/7 * Epoch 6 (test): loss=0.6945\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):   0% 0/32 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):   0% 0/32 [00:02<?, ?it/s, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):   3% 1/32 [00:02<01:18,  2.53s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):   3% 1/32 [00:05<01:18,  2.53s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):   6% 2/32 [00:05<01:16,  2.54s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):   6% 2/32 [00:07<01:16,  2.54s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):   9% 3/32 [00:07<01:14,  2.56s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):   9% 3/32 [00:10<01:14,  2.56s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  12% 4/32 [00:10<01:11,  2.57s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  12% 4/32 [00:12<01:11,  2.57s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  16% 5/32 [00:12<01:09,  2.57s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  16% 5/32 [00:15<01:09,  2.57s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  19% 6/32 [00:15<01:06,  2.56s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  19% 6/32 [00:17<01:06,  2.56s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  22% 7/32 [00:17<01:03,  2.55s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  22% 7/32 [00:20<01:03,  2.55s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  25% 8/32 [00:20<01:02,  2.58s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  25% 8/32 [00:23<01:02,  2.58s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  28% 9/32 [00:23<00:59,  2.58s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  28% 9/32 [00:25<00:59,  2.58s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  31% 10/32 [00:25<00:58,  2.64s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  31% 10/32 [00:28<00:58,  2.64s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  34% 11/32 [00:28<00:56,  2.68s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  34% 11/32 [00:31<00:56,  2.68s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  38% 12/32 [00:31<00:54,  2.72s/it, loss=0.699]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  38% 12/32 [00:34<00:54,  2.72s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  41% 13/32 [00:34<00:51,  2.72s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  41% 13/32 [00:36<00:51,  2.72s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  44% 14/32 [00:36<00:48,  2.68s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  44% 14/32 [00:39<00:48,  2.68s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  47% 15/32 [00:39<00:45,  2.66s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  47% 15/32 [00:42<00:45,  2.66s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  50% 16/32 [00:42<00:42,  2.67s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  50% 16/32 [00:44<00:42,  2.67s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  53% 17/32 [00:44<00:39,  2.64s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  53% 17/32 [00:47<00:39,  2.64s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  56% 18/32 [00:47<00:37,  2.66s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  56% 18/32 [00:49<00:37,  2.66s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  59% 19/32 [00:49<00:34,  2.63s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  59% 19/32 [00:52<00:34,  2.63s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  62% 20/32 [00:52<00:31,  2.61s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  62% 20/32 [00:55<00:31,  2.61s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  66% 21/32 [00:55<00:28,  2.61s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  66% 21/32 [00:57<00:28,  2.61s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  69% 22/32 [00:57<00:25,  2.60s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  69% 22/32 [01:00<00:25,  2.60s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  72% 23/32 [01:00<00:23,  2.60s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  72% 23/32 [01:02<00:23,  2.60s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  75% 24/32 [01:02<00:20,  2.62s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  75% 24/32 [01:05<00:20,  2.62s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  78% 25/32 [01:05<00:18,  2.62s/it, loss=0.700]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  78% 25/32 [01:08<00:18,  2.62s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  81% 26/32 [01:08<00:15,  2.60s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  81% 26/32 [01:10<00:15,  2.60s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  84% 27/32 [01:10<00:12,  2.58s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  84% 27/32 [01:13<00:12,  2.58s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  88% 28/32 [01:13<00:10,  2.56s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  88% 28/32 [01:15<00:10,  2.56s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  91% 29/32 [01:15<00:07,  2.54s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  91% 29/32 [01:18<00:07,  2.54s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  94% 30/32 [01:18<00:05,  2.53s/it, loss=0.698]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  94% 30/32 [01:20<00:05,  2.53s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  97% 31/32 [01:20<00:02,  2.56s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train):  97% 31/32 [01:23<00:02,  2.56s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train): 100% 32/32 [01:23<00:00,  2.57s/it, loss=0.697]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (train): 100% 32/32 [01:23<00:00,  2.61s/it, loss=0.697]\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (valid):   0% 0/2 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (valid):   0% 0/2 [00:02<?, ?it/s, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (valid):  50% 1/2 [00:02<00:02,  2.65s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (valid):  50% 1/2 [00:05<00:02,  2.65s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (valid): 100% 2/2 [00:05<00:00,  2.66s/it, loss=0.695]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (valid): 100% 2/2 [00:05<00:00,  2.67s/it, loss=0.695]\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (test):   0% 0/2 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (test):   0% 0/2 [00:02<?, ?it/s, loss=0.694]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (test):  50% 1/2 [00:02<00:02,  2.69s/it, loss=0.694]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (test):  50% 1/2 [00:05<00:02,  2.69s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (test): 100% 2/2 [00:05<00:00,  2.69s/it, loss=0.696]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "7/7 * Epoch (test): 100% 2/2 [00:05<00:00,  2.70s/it, loss=0.696]\n",
      "[2020-05-25 10:16:27,457] \n",
      "7/7 * Epoch 7 (_base): lr=0.0010 | momentum=0.9000\n",
      "7/7 * Epoch 7 (train): loss=0.6977\n",
      "7/7 * Epoch 7 (valid): loss=0.6951\n",
      "7/7 * Epoch 7 (test): loss=0.6945\n",
      "[2020-05-25 10:16:27,457] \n",
      "7/7 * Epoch 7 (_base): lr=0.0010 | momentum=0.9000\n",
      "7/7 * Epoch 7 (train): loss=0.6977\n",
      "7/7 * Epoch 7 (valid): loss=0.6951\n",
      "7/7 * Epoch 7 (test): loss=0.6945\n",
      "[2020-05-25 10:16:27,457] \n",
      "7/7 * Epoch 7 (_base): lr=0.0010 | momentum=0.9000\n",
      "7/7 * Epoch 7 (train): loss=0.6977\n",
      "7/7 * Epoch 7 (valid): loss=0.6951\n",
      "7/7 * Epoch 7 (test): loss=0.6945\n",
      "[2020-05-25 10:16:27,457] \n",
      "7/7 * Epoch 7 (_base): lr=0.0010 | momentum=0.9000\n",
      "7/7 * Epoch 7 (train): loss=0.6977\n",
      "7/7 * Epoch 7 (valid): loss=0.6951\n",
      "7/7 * Epoch 7 (test): loss=0.6945\n",
      "[2020-05-25 10:16:27,457] \n",
      "7/7 * Epoch 7 (_base): lr=0.0010 | momentum=0.9000\n",
      "7/7 * Epoch 7 (train): loss=0.6977\n",
      "7/7 * Epoch 7 (valid): loss=0.6951\n",
      "7/7 * Epoch 7 (test): loss=0.6945\n",
      "Top best models:\n",
      "logs/checkpoints/train.1.pth\t0.6951\n"
     ]
    }
   ],
   "source": [
    "import catalyst\n",
    "from catalyst.dl.runner import SupervisedRunner\n",
    "\n",
    "device = torch.device('cuda')\n",
    "\n",
    "loaders = {\n",
    "    'train': data.DataLoader(ExtDataset(ext_train_records), batch_size=128, collate_fn=collate_fn),\n",
    "    'valid': data.DataLoader(ExtDataset(ext_val_records), batch_size=128, collate_fn=collate_fn),\n",
    "    'test': data.DataLoader(ExtDataset(ext_test_records), batch_size=128, collate_fn=collate_fn),\n",
    "}\n",
    "\n",
    "lr = 1e-3\n",
    "num_epochs = 7\n",
    "\n",
    "optimizer  = torch.optim.Adam(model.parameters(), lr=lr)\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "runner = SupervisedRunner()\n",
    "runner.train(\n",
    "    model=model,\n",
    "    optimizer=optimizer,\n",
    "    loaders=loaders,\n",
    "    logdir='./logs',\n",
    "    num_epochs=num_epochs,\n",
    "    criterion=criterion,\n",
    "    verbose=True\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 329,
     "referenced_widgets": [
      "7edb4f59d633483bbf06b410d2348ce1",
      "4a376b412876472e93ac537ec297ef82",
      "5cdeeeff2a9e4865a422066b1a08c639",
      "a62c5b1c851e41e5a931f0037fb4de66",
      "82b08862ace74543bda12ef0193ef0a9",
      "1de25f0d67574c409b35224650e38278",
      "0f8c91a8da9a4ed3b69f3446b18971fe",
      "c9668480cefa404d908329d95d2d6e32"
     ]
    },
    "colab_type": "code",
    "id": "EwqhK2dyKuGL",
    "outputId": "b68ab17f-8c1b-40f7-acd4-b9fc02f99605"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:6: TqdmDeprecationWarning:\n",
      "\n",
      "This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7edb4f59d633483bbf06b410d2348ce1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=256.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:84: UserWarning:\n",
      "\n",
      "Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Count: 256\n",
      "Ref: очередной мост будет построен в россии. на этот раз через реку лена в районе якутска. смета составляет почти 83 млрд рублей. окупаемость моста вызывает сомнения из-за низкого трафика. это, скорее, социальный проект. и даже политический, учитывая положительные итоги строительства моста в крым.\n",
      "Hyp: президент россии владимир путин одобрил проект строительства моста через лену в якутске. в понедельник, 18 ноября, об этом заявил глава якутии айсен николаев , заявив, что одобрение от федеральных властей получено. ранее это стало известно по данным источников «коммерсанта», которые сообщили, что 9 ноября путин «фактически утвердил» проект строительства автодорожного моста через лену.\n",
      "BLEU:  0.4441327309711121\n",
      "ROUGE:  {'rouge-1': {'f': 0.2645632970896027, 'p': 0.25479653857679296, 'r': 0.2943782601001662}, 'rouge-2': {'f': 0.11298230206411987, 'p': 0.1069842755732323, 'r': 0.12953470673811784}, 'rouge-l': {'f': 0.22724253897875155, 'p': 0.23116526946756685, 'r': 0.2672520238629886}}\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda\")\n",
    "\n",
    "references = []\n",
    "predictions = []\n",
    "model.eval()\n",
    "for i, item in tqdm(enumerate(data.DataLoader(ExtDataset(ext_test_records), batch_size=1, collate_fn=collate_fn)), total=ext_test_records.shape[0]):\n",
    "    logits = model(item[\"features\"].to(device))[0] # Прямой проход\n",
    "    record = ext_test_records.iloc[i]\n",
    "    predicted_summary = []\n",
    "    for i, logit in enumerate(logits):\n",
    "        if logit > 0.0:\n",
    "            predicted_summary.append(record['sentences'][i])\n",
    "    if not predicted_summary:\n",
    "        predicted_summary.append(record['sentences'][torch.max(logits, dim=0)[1].item()])\n",
    "    predicted_summary = \" \".join(predicted_summary)\n",
    "    references.append(record['summary'].lower())\n",
    "    predictions.append(predicted_summary)\n",
    "\n",
    "calc_scores(references, predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4Kc0etEGfJ0p"
   },
   "outputs": [],
   "source": [
    "### Sources\n",
    "- https://arxiv.org/pdf/1611.04230.pdf\n",
    "\n",
    "\n",
    "- https://towardsdatascience.com/understanding-how-convolutional-neural-network-cnn-perform-text-classification-with-word-d2ee64b9dd0b Пример того, как можно применять CNN в текстовых задачах\n",
    "- https://arxiv.org/pdf/1808.08745.pdf Очень крутой метод генерации summary без Transformers\n",
    "- https://towardsdatascience.com/super-easy-way-to-get-sentence-embedding-using-fasttext-in-python-a70f34ac5b7c – простой метод генерации sentence embedding\n",
    "- https://towardsdatascience.com/fse-2b1ffa791cf9 – Необычный метод генерации sentence embedding\n",
    "- https://github.com/UKPLab/sentence-transformers – BERT предобученный для sentence embedding\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Final_[homework]TextSummarization.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "01bc56666c214eb4ad497074b3c94ab9": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "05f3590569264fb8ab3165fd71c6c22c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_b2555242f7ff4c758c4f50b5292ec2bb",
       "IPY_MODEL_f7f228f323e44697b234ac01850f68d0"
      ],
      "layout": "IPY_MODEL_8e4f8f6e414f4f57be5f2bdab68b05a0"
     }
    },
    "0732f432e8c34c3dbada9b2d4b48c6a0": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_4e413af1bc2b46d6ae78cdfee9a73c89",
      "placeholder": "​",
      "style": "IPY_MODEL_92a7a3bdfc0e470985a12802c51fd31d",
      "value": " 4096/4096 [04:51&lt;00:00, 14.06it/s]"
     }
    },
    "0f8c91a8da9a4ed3b69f3446b18971fe": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "0fd4eec972554953b90aedb4b00c5bca": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "105b402e7ece4df79cae6b720a80d210": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "11a778c6b88643df9f542240e99c84a4": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "1de25f0d67574c409b35224650e38278": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "22ff06420fca4347bd4ef0d8285a596d": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2494a76c1e484baa9c50214dae08f6c6": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ecb74cb487064e1cb3f7e1451886b1b3",
      "placeholder": "​",
      "style": "IPY_MODEL_39321aefe22844b0bf3f5b0306e344bd",
      "value": " 256/256 [00:26&lt;00:00,  9.50it/s]"
     }
    },
    "2e566e55ab0e4687af556fff5f990bb4": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "39321aefe22844b0bf3f5b0306e344bd": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "3a6749d40e1044828bc8f92095ae0c9d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_22ff06420fca4347bd4ef0d8285a596d",
      "max": 4096,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_81470fa4151f47d78102e74378ab79c1",
      "value": 4096
     }
    },
    "3e8cadbac2194b90b649ecce8b3651c3": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_db070b630e554e49abeed94367cdb882",
      "max": 256,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_62eaace7e0f44762a2c97be3b896d94e",
      "value": 256
     }
    },
    "4a376b412876472e93ac537ec297ef82": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4e413af1bc2b46d6ae78cdfee9a73c89": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "53c1a290165941c2b247885c5f8082bb": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "5cdeeeff2a9e4865a422066b1a08c639": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1de25f0d67574c409b35224650e38278",
      "max": 256,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_82b08862ace74543bda12ef0193ef0a9",
      "value": 256
     }
    },
    "62eaace7e0f44762a2c97be3b896d94e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "7b7306aa9b044908a13cffd959287717": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7edb4f59d633483bbf06b410d2348ce1": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_5cdeeeff2a9e4865a422066b1a08c639",
       "IPY_MODEL_a62c5b1c851e41e5a931f0037fb4de66"
      ],
      "layout": "IPY_MODEL_4a376b412876472e93ac537ec297ef82"
     }
    },
    "81470fa4151f47d78102e74378ab79c1": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "82b08862ace74543bda12ef0193ef0a9": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "8e4f8f6e414f4f57be5f2bdab68b05a0": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8e91ebac38b4493288c3cfd28c0959e4": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "92a7a3bdfc0e470985a12802c51fd31d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "a3a0111965bd4fd3852c9b096c875001": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "a4a1e862c16343aababbf404d6b56a5c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_105b402e7ece4df79cae6b720a80d210",
      "placeholder": "​",
      "style": "IPY_MODEL_53c1a290165941c2b247885c5f8082bb",
      "value": " 1000/1000 [01:22&lt;00:00, 12.16it/s]"
     }
    },
    "a62c5b1c851e41e5a931f0037fb4de66": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_c9668480cefa404d908329d95d2d6e32",
      "placeholder": "​",
      "style": "IPY_MODEL_0f8c91a8da9a4ed3b69f3446b18971fe",
      "value": " 256/256 [1:12:53&lt;00:00, 17.08s/it]"
     }
    },
    "b2555242f7ff4c758c4f50b5292ec2bb": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_7b7306aa9b044908a13cffd959287717",
      "max": 256,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_a3a0111965bd4fd3852c9b096c875001",
      "value": 256
     }
    },
    "b3f3a55a4ef142019faa85f0e110b017": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c08fbdfa99e54d15bb45254d23d53b1d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "c181c1cbad4c457d933f64048c305ce8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_3e8cadbac2194b90b649ecce8b3651c3",
       "IPY_MODEL_2494a76c1e484baa9c50214dae08f6c6"
      ],
      "layout": "IPY_MODEL_b3f3a55a4ef142019faa85f0e110b017"
     }
    },
    "c9668480cefa404d908329d95d2d6e32": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d10a80939c6c49b5b6fbe8103e69c1fc": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_01bc56666c214eb4ad497074b3c94ab9",
      "max": 1000,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_c08fbdfa99e54d15bb45254d23d53b1d",
      "value": 1000
     }
    },
    "db070b630e554e49abeed94367cdb882": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e44d5ff38cd944e5b5b4c9d225881562": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_d10a80939c6c49b5b6fbe8103e69c1fc",
       "IPY_MODEL_a4a1e862c16343aababbf404d6b56a5c"
      ],
      "layout": "IPY_MODEL_2e566e55ab0e4687af556fff5f990bb4"
     }
    },
    "e97deeb0dbd34763918da3cd1ce2d96d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_3a6749d40e1044828bc8f92095ae0c9d",
       "IPY_MODEL_0732f432e8c34c3dbada9b2d4b48c6a0"
      ],
      "layout": "IPY_MODEL_0fd4eec972554953b90aedb4b00c5bca"
     }
    },
    "ecb74cb487064e1cb3f7e1451886b1b3": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f7f228f323e44697b234ac01850f68d0": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_8e91ebac38b4493288c3cfd28c0959e4",
      "placeholder": "​",
      "style": "IPY_MODEL_11a778c6b88643df9f542240e99c84a4",
      "value": " 256/256 [00:23&lt;00:00, 11.08it/s]"
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}