baseline.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"authorship_tag":"ABX9TyNCAhMVy7notjS3NuqeWhhV"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":2,"metadata":{"id":"WvnJ1j94omdm","executionInfo":{"status":"ok","timestamp":1674393956364,"user_tz":-180,"elapsed":280,"user":{"displayName":"Anton Golubev","userId":"02823576701835399427"}}},"outputs":[],"source":["\"\"\"PyTorch optimization for BERT model.\"\"\"\n","\n","import math\n","import torch\n","from torch.nn.utils import clip_grad_norm_\n","from torch.optim import Optimizer\n","\n","\n","def warmup_cosine(x, warmup=0.002):\n","    if x < warmup:\n","        return x / warmup\n","    return 0.5 * (1.0 + torch.cos(math.pi * x))\n","\n","\n","def warmup_constant(x, warmup=0.002):\n","    if x < warmup:\n","        return x / warmup\n","    return 1.0\n","\n","\n","def warmup_linear(x, warmup=0.002):\n","    if x < warmup:\n","        return x / warmup\n","    return 1.0 - x\n","\n","\n","SCHEDULES = {\n","    'warmup_cosine': warmup_cosine,\n","    'warmup_constant': warmup_constant,\n","    'warmup_linear': warmup_linear,\n","}\n","\n","\n","class BERTAdam(Optimizer):\n","\n","    def __init__(self, params, lr, warmup=-1, t_total=-1, schedule='warmup_linear',\n","                 b1=0.9, b2=0.999, e=1e-6, weight_decay_rate=0.01,\n","                 max_grad_norm=1.0):\n","        if not lr >= 0.0:\n","            raise ValueError(\"Invalid learning rate: {} - should be >= 0.0\".format(lr))\n","        if schedule not in SCHEDULES:\n","            raise ValueError(\"Invalid schedule parameter: {}\".format(schedule))\n","        if not 0.0 <= warmup < 1.0 and not warmup == -1:\n","            raise ValueError(\"Invalid warmup: {} - should be in [0.0, 1.0[ or -1\".format(warmup))\n","        if not 0.0 <= b1 < 1.0:\n","            raise ValueError(\"Invalid b1 parameter: {} - should be in [0.0, 1.0[\".format(b1))\n","        if not 0.0 <= b2 < 1.0:\n","            raise ValueError(\"Invalid b2 parameter: {} - should be in [0.0, 1.0[\".format(b2))\n","        if not e >= 0.0:\n","            raise ValueError(\"Invalid epsilon value: {} - should be >= 0.0\".format(e))\n","        defaults = dict(lr=lr, schedule=schedule, warmup=warmup, t_total=t_total,\n","                        b1=b1, b2=b2, e=e, weight_decay_rate=weight_decay_rate,\n","                        max_grad_norm=max_grad_norm)\n","        super(BERTAdam, self).__init__(params, defaults)\n","\n","    def get_lr(self):\n","        lr = []\n","        print(\"l_total=\", len(self.param_groups))\n","        for group in self.param_groups:\n","            print(\"l_p=\", len(group['params']))\n","            for p in group['params']:\n","                state = self.state[p]\n","                if len(state) == 0:\n","                    return [0]\n","                if group['t_total'] != -1:\n","                    schedule_fct = SCHEDULES[group['schedule']]\n","                    lr_scheduled = group['lr'] * schedule_fct(state['step'] / group['t_total'], group['warmup'])\n","                else:\n","                    lr_scheduled = group['lr']\n","                lr.append(lr_scheduled)\n","        return lr\n","\n","    def to(self, device):\n","        \"\"\" Move the optimizer state to a specified device\"\"\"\n","        for state in self.state.values():\n","            state['exp_avg'].to(device)\n","            state['exp_avg_sq'].to(device)\n","\n","    def initialize_step(self, initial_step):\n","        for group in self.param_groups:\n","            for p in group['params']:\n","                state = self.state[p]\n","                state['step'] = initial_step\n","                state['exp_avg'] = torch.zeros_like(p.data)\n","                state['exp_avg_sq'] = torch.zeros_like(p.data)\n","\n","    def step(self, closure=None):\n","        loss = None\n","        if closure is not None:\n","            loss = closure()\n","\n","        for group in self.param_groups:\n","            for p in group['params']:\n","                if p.grad is None:\n","                    continue\n","                grad = p.grad.data\n","                if grad.is_sparse:\n","                    raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead')\n","\n","                state = self.state[p]\n","\n","                if len(state) == 0:\n","                    state['step'] = 0\n","                    state['next_m'] = torch.zeros_like(p.data)\n","                    state['next_v'] = torch.zeros_like(p.data)\n","\n","                next_m, next_v = state['next_m'], state['next_v']\n","                beta1, beta2 = group['b1'], group['b2']\n","\n","                if group['max_grad_norm'] > 0:\n","                    clip_grad_norm_(p, group['max_grad_norm'])\n","\n","                next_m.mul_(beta1).add_(1 - beta1, grad)\n","                next_v.mul_(beta2).addcmul_(1 - beta2, grad, grad)\n","                update = next_m / (next_v.sqrt() + group['e'])\n","\n","                if group['weight_decay_rate'] > 0.0:\n","                    update += group['weight_decay_rate'] * p.data\n","\n","                if group['t_total'] != -1:\n","                    schedule_fct = SCHEDULES[group['schedule']]\n","                    lr_scheduled = group['lr'] * schedule_fct(state['step'] / group['t_total'], group['warmup'])\n","                else:\n","                    lr_scheduled = group['lr']\n","\n","                update_with_lr = lr_scheduled * update\n","                p.data.add_(-update_with_lr)\n","\n","                state['step'] += 1\n","\n","        return loss"]},{"cell_type":"code","source":["\"\"\"PyTorch BERT model.\"\"\"\n","\n","from __future__ import absolute_import, division, print_function\n","\n","import copy\n","import json\n","import math\n","\n","import six\n","import torch\n","import torch.nn as nn\n","from torch.nn import CrossEntropyLoss\n","\n","\n","def gelu(x):\n","    \"\"\"Implementation of the gelu activation function.\n","        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):\n","        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))\n","    \"\"\"\n","    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))\n","\n","\n","class BertConfig(object):\n","    \"\"\"Configuration class to store the configuration of a `BertModel`.\n","    \"\"\"\n","\n","    def __init__(self,\n","                 vocab_size,\n","                 hidden_size=768,\n","                 num_hidden_layers=12,\n","                 num_attention_heads=12,\n","                 intermediate_size=3072,\n","                 hidden_act=\"gelu\",\n","                 hidden_dropout_prob=0.1,\n","                 attention_probs_dropout_prob=0.1,\n","                 max_position_embeddings=512,\n","                 type_vocab_size=16,\n","                 initializer_range=0.02):\n","\n","        self.vocab_size = vocab_size\n","        self.hidden_size = hidden_size\n","        self.num_hidden_layers = num_hidden_layers\n","        self.num_attention_heads = num_attention_heads\n","        self.hidden_act = hidden_act\n","        self.intermediate_size = intermediate_size\n","        self.hidden_dropout_prob = hidden_dropout_prob\n","        self.attention_probs_dropout_prob = attention_probs_dropout_prob\n","        self.max_position_embeddings = max_position_embeddings\n","        self.type_vocab_size = type_vocab_size\n","        self.initializer_range = initializer_range\n","\n","    @classmethod\n","    def from_dict(cls, json_object):\n","        \"\"\"Constructs a `BertConfig` from a Python dictionary of parameters.\"\"\"\n","        config = BertConfig(vocab_size=None)\n","        for (key, value) in six.iteritems(json_object):\n","            config.__dict__[key] = value\n","        return config\n","\n","    @classmethod\n","    def from_json_file(cls, json_file):\n","        \"\"\"Constructs a `BertConfig` from a json file of parameters.\"\"\"\n","        with open(json_file, \"r\") as reader:\n","            text = reader.read()\n","        return cls.from_dict(json.loads(text))\n","\n","    def to_dict(self):\n","        \"\"\"Serializes this instance to a Python dictionary.\"\"\"\n","        output = copy.deepcopy(self.__dict__)\n","        return output\n","\n","    def to_json_string(self):\n","        \"\"\"Serializes this instance to a JSON string.\"\"\"\n","        return json.dumps(self.to_dict(), indent=2, sort_keys=True) + \"\\n\"\n","\n","\n","class BERTLayerNorm(nn.Module):\n","    def __init__(self, config, variance_epsilon=1e-12):\n","        \"\"\"Construct a layernorm module in the TF style (epsilon inside the square root).\n","        \"\"\"\n","        super(BERTLayerNorm, self).__init__()\n","        self.gamma = nn.Parameter(torch.ones(config.hidden_size))\n","        self.beta = nn.Parameter(torch.zeros(config.hidden_size))\n","        self.variance_epsilon = variance_epsilon\n","\n","    def forward(self, x):\n","        u = x.mean(-1, keepdim=True)\n","        s = (x - u).pow(2).mean(-1, keepdim=True)\n","        x = (x - u) / torch.sqrt(s + self.variance_epsilon)\n","        return self.gamma * x + self.beta\n","\n","\n","class BERTEmbeddings(nn.Module):\n","    def __init__(self, config):\n","        super(BERTEmbeddings, self).__init__()\n","        \"\"\"Construct the embedding module from word, position and token_type embeddings.\n","        \"\"\"\n","        self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size)\n","        self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size)\n","        self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.hidden_size)\n","        self.LayerNorm = BERTLayerNorm(config)\n","        self.dropout = nn.Dropout(config.hidden_dropout_prob)\n","\n","    def forward(self, input_ids, token_type_ids=None):\n","        seq_length = input_ids.size(1)\n","        position_ids = torch.arange(seq_length, dtype=torch.long, device=input_ids.device)\n","        position_ids = position_ids.unsqueeze(0).expand_as(input_ids)\n","        if token_type_ids is None:\n","            token_type_ids = torch.zeros_like(input_ids)\n","\n","        words_embeddings = self.word_embeddings(input_ids)\n","        position_embeddings = self.position_embeddings(position_ids)\n","        token_type_embeddings = self.token_type_embeddings(token_type_ids)\n","\n","        embeddings = words_embeddings + position_embeddings + token_type_embeddings\n","        embeddings = self.LayerNorm(embeddings)\n","        embeddings = self.dropout(embeddings)\n","        return embeddings\n","\n","\n","class BERTSelfAttention(nn.Module):\n","    def __init__(self, config):\n","        super(BERTSelfAttention, self).__init__()\n","        if config.hidden_size % config.num_attention_heads != 0:\n","            raise ValueError(\n","                \"The hidden size (%d) is not a multiple of the number of attention \"\n","                \"heads (%d)\" % (config.hidden_size, config.num_attention_heads))\n","        self.num_attention_heads = config.num_attention_heads\n","        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)\n","        self.all_head_size = self.num_attention_heads * self.attention_head_size\n","\n","        self.query = nn.Linear(config.hidden_size, self.all_head_size)\n","        self.key = nn.Linear(config.hidden_size, self.all_head_size)\n","        self.value = nn.Linear(config.hidden_size, self.all_head_size)\n","\n","        self.dropout = nn.Dropout(config.attention_probs_dropout_prob)\n","\n","    def transpose_for_scores(self, x):\n","        new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size)\n","        x = x.view(*new_x_shape)\n","        return x.permute(0, 2, 1, 3)\n","\n","    def forward(self, hidden_states, attention_mask):\n","        mixed_query_layer = self.query(hidden_states)\n","        mixed_key_layer = self.key(hidden_states)\n","        mixed_value_layer = self.value(hidden_states)\n","\n","        query_layer = self.transpose_for_scores(mixed_query_layer)\n","        key_layer = self.transpose_for_scores(mixed_key_layer)\n","        value_layer = self.transpose_for_scores(mixed_value_layer)\n","\n","        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))\n","        attention_scores = attention_scores / math.sqrt(self.attention_head_size)\n","        attention_scores = attention_scores + attention_mask\n","        attention_probs = nn.Softmax(dim=-1)(attention_scores)\n","        attention_probs = self.dropout(attention_probs)\n","        context_layer = torch.matmul(attention_probs, value_layer)\n","        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()\n","        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)\n","        context_layer = context_layer.view(*new_context_layer_shape)\n","        return context_layer\n","\n","\n","class BERTSelfOutput(nn.Module):\n","    def __init__(self, config):\n","        super(BERTSelfOutput, self).__init__()\n","        self.dense = nn.Linear(config.hidden_size, config.hidden_size)\n","        self.LayerNorm = BERTLayerNorm(config)\n","        self.dropout = nn.Dropout(config.hidden_dropout_prob)\n","\n","    def forward(self, hidden_states, input_tensor):\n","        hidden_states = self.dense(hidden_states)\n","        hidden_states = self.dropout(hidden_states)\n","        hidden_states = self.LayerNorm(hidden_states + input_tensor)\n","        return hidden_states\n","\n","\n","class BERTAttention(nn.Module):\n","    def __init__(self, config):\n","        super(BERTAttention, self).__init__()\n","        self.self = BERTSelfAttention(config)\n","        self.output = BERTSelfOutput(config)\n","\n","    def forward(self, input_tensor, attention_mask):\n","        self_output = self.self(input_tensor, attention_mask)\n","        attention_output = self.output(self_output, input_tensor)\n","        return attention_output\n","\n","\n","class BERTIntermediate(nn.Module):\n","    def __init__(self, config):\n","        super(BERTIntermediate, self).__init__()\n","        self.dense = nn.Linear(config.hidden_size, config.intermediate_size)\n","        self.intermediate_act_fn = gelu\n","\n","    def forward(self, hidden_states):\n","        hidden_states = self.dense(hidden_states)\n","        hidden_states = self.intermediate_act_fn(hidden_states)\n","        return hidden_states\n","\n","\n","class BERTOutput(nn.Module):\n","    def __init__(self, config):\n","        super(BERTOutput, self).__init__()\n","        self.dense = nn.Linear(config.intermediate_size, config.hidden_size)\n","        self.LayerNorm = BERTLayerNorm(config)\n","        self.dropout = nn.Dropout(config.hidden_dropout_prob)\n","\n","    def forward(self, hidden_states, input_tensor):\n","        hidden_states = self.dense(hidden_states)\n","        hidden_states = self.dropout(hidden_states)\n","        hidden_states = self.LayerNorm(hidden_states + input_tensor)\n","        return hidden_states\n","\n","\n","class BERTLayer(nn.Module):\n","    def __init__(self, config):\n","        super(BERTLayer, self).__init__()\n","        self.attention = BERTAttention(config)\n","        self.intermediate = BERTIntermediate(config)\n","        self.output = BERTOutput(config)\n","\n","    def forward(self, hidden_states, attention_mask):\n","        attention_output = self.attention(hidden_states, attention_mask)\n","        intermediate_output = self.intermediate(attention_output)\n","        layer_output = self.output(intermediate_output, attention_output)\n","        return layer_output\n","\n","\n","class BERTEncoder(nn.Module):\n","    def __init__(self, config):\n","        super(BERTEncoder, self).__init__()\n","        layer = BERTLayer(config)\n","        self.layer = nn.ModuleList([copy.deepcopy(layer) for _ in range(config.num_hidden_layers)])\n","\n","    def forward(self, hidden_states, attention_mask):\n","        all_encoder_layers = []\n","        for layer_module in self.layer:\n","            hidden_states = layer_module(hidden_states, attention_mask)\n","            all_encoder_layers.append(hidden_states)\n","        return all_encoder_layers\n","\n","\n","class BERTPooler(nn.Module):\n","    def __init__(self, config):\n","        super(BERTPooler, self).__init__()\n","        self.dense = nn.Linear(config.hidden_size, config.hidden_size)\n","        self.activation = nn.Tanh()\n","\n","    def forward(self, hidden_states):\n","        first_token_tensor = hidden_states[:, 0]\n","        pooled_output = self.dense(first_token_tensor)\n","        pooled_output = self.activation(pooled_output)\n","        return pooled_output\n","\n","\n","class BertModel(nn.Module):\n","\n","    def __init__(self, config: BertConfig):\n","        super(BertModel, self).__init__()\n","        self.embeddings = BERTEmbeddings(config)\n","        self.encoder = BERTEncoder(config)\n","        self.pooler = BERTPooler(config)\n","\n","    def forward(self, input_ids, token_type_ids=None, attention_mask=None):\n","        if attention_mask is None:\n","            attention_mask = torch.ones_like(input_ids)\n","        if token_type_ids is None:\n","            token_type_ids = torch.zeros_like(input_ids)\n","\n","        extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2)\n","\n","        extended_attention_mask = extended_attention_mask.float()\n","        extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0\n","\n","        embedding_output = self.embeddings(input_ids, token_type_ids)\n","        all_encoder_layers = self.encoder(embedding_output, extended_attention_mask)\n","        sequence_output = all_encoder_layers[-1]\n","        pooled_output = self.pooler(sequence_output)\n","        return all_encoder_layers, pooled_output\n","\n","\n","class BertForSequenceClassification(nn.Module):\n","\n","    def __init__(self, config, num_labels):\n","        super(BertForSequenceClassification, self).__init__()\n","        self.bert = BertModel(config)\n","        self.dropout = nn.Dropout(config.hidden_dropout_prob)\n","        self.classifier = nn.Linear(config.hidden_size, num_labels)\n","\n","        def init_weights(module):\n","            if isinstance(module, (nn.Linear, nn.Embedding)):\n","                module.weight.data.normal_(mean=0.0, std=config.initializer_range)\n","            elif isinstance(module, BERTLayerNorm):\n","                module.beta.data.normal_(mean=0.0, std=config.initializer_range)\n","                module.gamma.data.normal_(mean=0.0, std=config.initializer_range)\n","            if isinstance(module, nn.Linear):\n","                module.bias.data.zero_()\n","\n","        self.apply(init_weights)\n","\n","    def forward(self, input_ids, token_type_ids, attention_mask, labels=None):\n","        _, pooled_output = self.bert(input_ids, token_type_ids, attention_mask)\n","        pooled_output = self.dropout(pooled_output)\n","        logits = self.classifier(pooled_output)\n","\n","        if labels is not None:\n","            loss_fct = CrossEntropyLoss()\n","            loss = loss_fct(logits, labels)\n","            return loss, logits\n","        else:\n","            return logits\n"],"metadata":{"id":"gpMcYsoxpzbV","executionInfo":{"status":"ok","timestamp":1674394078144,"user_tz":-180,"elapsed":679,"user":{"displayName":"Anton Golubev","userId":"02823576701835399427"}}},"execution_count":3,"outputs":[]},{"cell_type":"code","source":["\"\"\"Tokenization classes.\"\"\"\n","\n","from __future__ import absolute_import, division, print_function\n","\n","import collections\n","import unicodedata\n","\n","import six\n","\n","\n","def convert_to_unicode(text):\n","    \"\"\"Converts `text` to Unicode (if it's not already), assuming utf-8 input.\"\"\"\n","    if six.PY3:\n","        if isinstance(text, str):\n","            return text\n","        elif isinstance(text, bytes):\n","            return text.decode(\"utf-8\", \"ignore\")\n","        else:\n","            raise ValueError(\"Unsupported string type: %s\" % (type(text)))\n","    elif six.PY2:\n","        if isinstance(text, str):\n","            return text.decode(\"utf-8\", \"ignore\")\n","        elif isinstance(text, unicode):\n","            return text\n","        else:\n","            raise ValueError(\"Unsupported string type: %s\" % (type(text)))\n","    else:\n","        raise ValueError(\"Not running on Python2 or Python 3?\")\n","\n","\n","def printable_text(text):\n","    \"\"\"Returns text encoded in a way suitable for print or `tf.logging`.\"\"\"\n","\n","    if six.PY3:\n","        if isinstance(text, str):\n","            return text\n","        elif isinstance(text, bytes):\n","            return text.decode(\"utf-8\", \"ignore\")\n","        else:\n","            raise ValueError(\"Unsupported string type: %s\" % (type(text)))\n","    elif six.PY2:\n","        if isinstance(text, str):\n","            return text\n","        elif isinstance(text, unicode):\n","            return text.encode(\"utf-8\")\n","        else:\n","            raise ValueError(\"Unsupported string type: %s\" % (type(text)))\n","    else:\n","        raise ValueError(\"Not running on Python2 or Python 3?\")\n","\n","\n","def load_vocab(vocab_file):\n","    \"\"\"Loads a vocabulary file into a dictionary.\n","    (строка_0, 0) ... (строка_n, n)\n","    \"\"\"\n","    vocab = collections.OrderedDict()\n","    index = 0\n","    with open(vocab_file, \"r\", encoding='utf-8') as reader:\n","        while True:\n","            token = convert_to_unicode(reader.readline())\n","            if not token:\n","                break\n","            token = token.strip()\n","            vocab[token] = index\n","            index += 1\n","    return vocab\n","\n","\n","def convert_tokens_to_ids(vocab, tokens):\n","    \"\"\"Converts a sequence of tokens into ids using the vocab.\n","    (если токена в словаре нет - ошибка)\"\"\"\n","    ids = []\n","    for token in tokens:\n","        ids.append(vocab[token])\n","    return ids\n","\n","\n","def whitespace_tokenize(text):\n","    \"\"\"Runs basic whitespace cleaning and splitting on a piece of text.\"\"\"\n","    text = text.strip()\n","    if not text:\n","        return []\n","    tokens = text.split()\n","    return tokens\n","\n","\n","class FullTokenizer(object):\n","    \"\"\"Runs end-to-end tokenization.\n","    разбивает предложение по кускам слов\"\"\"\n","\n","    def __init__(self, vocab_file, do_lower_case=True):\n","        self.vocab = load_vocab(vocab_file)\n","        self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case)\n","        self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab)\n","\n","    def tokenize(self, text):\n","        split_tokens = []\n","        for token in self.basic_tokenizer.tokenize(text):\n","            for sub_token in self.wordpiece_tokenizer.tokenize(token):\n","                split_tokens.append(sub_token)\n","\n","        return split_tokens\n","\n","    def convert_tokens_to_ids(self, tokens):\n","        return convert_tokens_to_ids(self.vocab, tokens)\n","\n","\n","class BasicTokenizer(object):\n","    \"\"\"Runs basic tokenization (punctuation splitting, lower casing, etc.).\"\"\"\n","\n","    def __init__(self, do_lower_case=True):\n","        \"\"\"Constructs a BasicTokenizer.\n","\n","        Args:\n","          do_lower_case: Whether to lower case the input.\n","        \"\"\"\n","        self.do_lower_case = do_lower_case\n","\n","    def tokenize(self, text):\n","        \"\"\"Tokenizes a piece of text.\"\"\"\n","        text = convert_to_unicode(text)\n","        text = self._clean_text(text)\n","        orig_tokens = whitespace_tokenize(text)\n","        split_tokens = []\n","        for token in orig_tokens:\n","            if self.do_lower_case:\n","                token = token.lower()\n","                token = self._run_strip_accents(token)\n","            split_tokens.extend(self._run_split_on_punc(token))\n","\n","        output_tokens = whitespace_tokenize(\" \".join(split_tokens))\n","        return output_tokens\n","\n","    def _run_strip_accents(self, text):\n","        \"\"\"Strips accents from a piece of text.\"\"\"\n","        text = unicodedata.normalize(\"NFD\", text)\n","        output = []\n","        for char in text:\n","            cat = unicodedata.category(char)\n","            if cat == \"Mn\":\n","                continue\n","            output.append(char)\n","        return \"\".join(output)\n","\n","    def _run_split_on_punc(self, text):\n","        \"\"\"Splits punctuation on a piece of text.\n","        вытаскивает знаки пунктуации в отдельные токены\"\"\"\n","        chars = list(text)\n","        i = 0\n","        start_new_word = True\n","        output = []\n","        while i < len(chars):\n","            char = chars[i]\n","            if _is_punctuation(char):\n","                output.append([char])\n","                start_new_word = True\n","            else:\n","                if start_new_word:\n","                    output.append([])\n","                start_new_word = False\n","                output[-1].append(char)\n","            i += 1\n","\n","        return [\"\".join(x) for x in output]\n","\n","    def _clean_text(self, text):\n","        \"\"\"Performs invalid character removal and whitespace cleanup on text.\"\"\"\n","        output = []\n","        for char in text:\n","            cp = ord(char)\n","            if cp == 0 or cp == 0xfffd or _is_control(char):\n","                continue\n","            if _is_whitespace(char):\n","                output.append(\" \")\n","            else:\n","                output.append(char)\n","        return \"\".join(output)\n","\n","\n","class WordpieceTokenizer(object):\n","    \"\"\"Runs WordPiece tokenization.\"\"\"\n","\n","    def __init__(self, vocab, unk_token=\"[UNK]\", max_input_chars_per_word=100):\n","        self.vocab = vocab\n","        self.unk_token = unk_token\n","        self.max_input_chars_per_word = max_input_chars_per_word\n","\n","    def tokenize(self, text):\n","        text = convert_to_unicode(text)\n","\n","        output_tokens = []\n","        for token in whitespace_tokenize(text):\n","            chars = list(token)\n","            if len(chars) > self.max_input_chars_per_word:\n","                output_tokens.append(self.unk_token)\n","                continue\n","\n","            is_bad = False\n","            start = 0\n","            sub_tokens = []\n","            while start < len(chars):\n","                end = len(chars)\n","                cur_substr = None\n","                while start < end:\n","                    substr = \"\".join(chars[start:end])\n","                    if start > 0:\n","                        substr = \"##\" + substr\n","                    if substr in self.vocab:\n","                        cur_substr = substr\n","                        break\n","                    end -= 1\n","                if cur_substr is None:\n","                    is_bad = True\n","                    break\n","                sub_tokens.append(cur_substr)\n","                start = end\n","\n","            if is_bad:\n","                output_tokens.append(self.unk_token)\n","            else:\n","                output_tokens.extend(sub_tokens)\n","        return output_tokens\n","\n","\n","def _is_whitespace(char):\n","    \"\"\"Checks whether `chars` is a whitespace character.\"\"\"\n","    if char == \" \" or char == \"\\t\" or char == \"\\n\" or char == \"\\r\":\n","        return True\n","    cat = unicodedata.category(char)\n","    if cat == \"Zs\":\n","        return True\n","    return False\n","\n","\n","def _is_control(char):\n","    \"\"\"Checks whether `chars` is a control character.\n","    осталось узнать что такое control character\"\"\"\n","    if char == \"\\t\" or char == \"\\n\" or char == \"\\r\":\n","        return False\n","    cat = unicodedata.category(char)\n","    if cat.startswith(\"C\"):\n","        return True\n","    return False\n","\n","\n","def _is_punctuation(char):\n","    \"\"\"Checks whether `chars` is a punctuation character.\n","    все специальные знаки с клавиатуры кроме пробела\"\"\"\n","    cp = ord(char)\n","    if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or\n","            (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)):\n","        return True\n","    cat = unicodedata.category(char)\n","    if cat.startswith(\"P\"):\n","        return True\n","    return False"],"metadata":{"id":"_eWlQT5nuPbk","executionInfo":{"status":"ok","timestamp":1674395111803,"user_tz":-180,"elapsed":12,"user":{"displayName":"Anton Golubev","userId":"02823576701835399427"}}},"execution_count":4,"outputs":[]},{"cell_type":"code","source":["\"\"\"Processor\"\"\"\n","\n","import csv\n","import os\n","import pandas as pd\n","\n","class InputExample(object):\n","\n","    def __init__(self, guid, text_a, text_b=None, label=None):\n","        self.guid = guid\n","        self.text_a = text_a\n","        self.text_b = text_b\n","        self.label = label\n","\n","\n","class DataProcessor(object):\n","    \"\"\"Base class for data converters for sequence classification data sets.\"\"\"\n","\n","    def get_train_examples(self, data_dir):\n","        \"\"\"Gets a collection of `InputExample`s for the train set.\"\"\"\n","        raise NotImplementedError()\n","\n","    def get_dev_examples(self, data_dir):\n","        \"\"\"Gets a collection of `InputExample`s for the dev set.\"\"\"\n","        raise NotImplementedError()\n","\n","    def get_test_examples(self, data_dir):\n","        \"\"\"Gets a collection of `InputExample`s for the test set.\"\"\"\n","        raise NotImplementedError()\n","\n","    def get_labels(self):\n","        \"\"\"Gets the list of labels for this data set.\"\"\"\n","        raise NotImplementedError()\n","\n","    @classmethod\n","    def _read_tsv(cls, input_file, quotechar=None):\n","        \"\"\"Reads a tab separated value file.\"\"\"\n","        with open(input_file, \"r\") as f:\n","            reader = csv.reader(f, delimiter=\"\\t\", quotechar=quotechar)\n","            lines = []\n","            for line in reader:\n","                lines.append(line)\n","            return lines\n","\n","\n","class Dataset_single_Processor(DataProcessor):\n","    \"\"\"Processor for the\n","    Sentihood data set.\"\"\"\n","\n","    def get_train_examples(self, data_dir):\n","        \"\"\"See base class.\"\"\"\n","        train_data = pd.read_csv(os.path.join(data_dir, \"train.tsv\"), header=0, sep=\"\\t\").values\n","        return self._create_examples(train_data, \"train\")\n","\n","    def get_dev_examples(self, data_dir):\n","        \"\"\"See base class.\"\"\"\n","        dev_data = pd.read_csv(os.path.join(data_dir, \"dev.tsv\"), header=0, sep=\"\\t\").values\n","        return self._create_examples(dev_data, \"dev\")\n","\n","    def get_test_examples(self, data_dir):\n","        \"\"\"See base class.\"\"\"\n","        test_data = pd.read_csv(os.path.join(data_dir, \"test.tsv\"), header=0, sep=\"\\t\").values\n","        return self._create_examples(test_data, \"test\")\n","\n","    def get_labels(self):\n","        \"\"\"See base class.\"\"\"\n","        return ['положительно', 'отрицательно', 'нейтрально']\n","\n","    def _create_examples(self, lines, set_type):\n","        \"\"\"Creates examples for the training and dev sets.\"\"\"\n","        examples = []\n","        for (i, line) in enumerate(lines):\n","            #  if i>50:break\n","            guid = \"%s-%s\" % (set_type, i)\n","            text_a = tokenization.convert_to_unicode(str(line[1]))\n","            label = tokenization.convert_to_unicode(str(line[2]))\n","            if i % 1000 == 0:\n","                print(i)\n","                print(\"guid=\", guid)\n","                print(\"text_a=\", text_a)\n","                print(\"label=\", label)\n","            examples.append(\n","                InputExample(guid=guid, text_a=text_a, text_b=None, label=label))\n","        return examples"],"metadata":{"id":"fFXKj-2OqgZz","executionInfo":{"status":"ok","timestamp":1674395148544,"user_tz":-180,"elapsed":2,"user":{"displayName":"Anton Golubev","userId":"02823576701835399427"}}},"execution_count":6,"outputs":[]},{"cell_type":"code","source":["from __future__ import absolute_import, division, print_function\n","\n","import argparse\n","import collections\n","import logging\n","import os\n","import random\n","import numpy as np\n","import pandas as pd\n","import torch\n","import torch.nn.functional as F\n","\n","from torch.utils.data import DataLoader, TensorDataset\n","from torch.utils.data.distributed import DistributedSampler\n","from torch.utils.data.sampler import RandomSampler\n","from tqdm import tqdm, trange\n","from sklearn.decomposition import PCA as sklearnPCA\n","from collections import Counter\n","\n","\n","logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',\n","                    datefmt='%m/%d/%Y %H:%M:%S',\n","                    level=logging.INFO)\n","logger = logging.getLogger(__name__)\n","\n","\n","class InputFeatures(object):\n","    \"\"\"A single set of features of data.\"\"\"\n","\n","    def __init__(self, input_ids, input_mask, segment_ids, label_id):\n","        self.input_ids = input_ids\n","        self.input_mask = input_mask\n","        self.segment_ids = segment_ids\n","        self.label_id = label_id\n","\n","\n","def convert_examples_to_features(examples, label_list, max_seq_length, tokenizer):\n","    \"\"\"Loads a data file into a list of `InputBatch`s.\n","    train_examples = список экземпляров класса InputExample (guid, text_a, text_b, label)\n","    label_list = метки категорий (pos/neg/conflict...)\n","    max_seq_length = гиперпараметр\"\"\"\n","\n","    label_map = {}\n","    for (i, label) in enumerate(label_list):\n","        label_map[label] = i\n","\n","    features = []\n","    for (ex_index, example) in enumerate(tqdm(examples)):\n","        tokens_a = tokenizer.tokenize(example.text_a)\n","\n","        tokens_b = None\n","        if example.text_b:\n","            tokens_b = tokenizer.tokenize(example.text_b)\n","\n","        if tokens_b:\n","            _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3)\n","        else:\n","            if len(tokens_a) > max_seq_length - 2:\n","                tokens_a = tokens_a[0:(max_seq_length - 2)]\n","\n","        tokens = []\n","        segment_ids = []\n","        tokens.append(\"[CLS]\")\n","        segment_ids.append(0)\n","        for token in tokens_a:\n","            tokens.append(token)\n","            segment_ids.append(0)\n","        tokens.append(\"[SEP]\")\n","        segment_ids.append(0)\n","\n","        if tokens_b:\n","            for token in tokens_b:\n","                tokens.append(token)\n","                segment_ids.append(1)\n","            tokens.append(\"[SEP]\")\n","            segment_ids.append(1)\n","\n","        input_ids = tokenizer.convert_tokens_to_ids(tokens)\n","        input_mask = [1] * len(input_ids)\n","\n","        while len(input_ids) < max_seq_length:\n","            input_ids.append(0)\n","            input_mask.append(0)\n","            segment_ids.append(0)\n","\n","        assert len(input_ids) == max_seq_length\n","        assert len(input_mask) == max_seq_length\n","        assert len(segment_ids) == max_seq_length\n","\n","        label_id = label_map[example.label]\n","\n","        features.append(InputFeatures(\n","            input_ids=input_ids,\n","            input_mask=input_mask, \n","            segment_ids=segment_ids, \n","            label_id=label_id) \n","        )\n","    return features\n","\n","\n","def _truncate_seq_pair(tokens_a, tokens_b, max_length):\n","    \"\"\"Truncates a sequence pair in place to the maximum length.\"\"\"\n","\n","    while True:\n","        total_length = len(tokens_a) + len(tokens_b)\n","        if total_length <= max_length:\n","            break\n","        if len(tokens_a) > len(tokens_b):\n","            tokens_a.pop()\n","        else:\n","            tokens_b.pop()"],"metadata":{"id":"oifM9MFrujZq","executionInfo":{"status":"ok","timestamp":1674396175853,"user_tz":-180,"elapsed":6,"user":{"displayName":"Anton Golubev","userId":"02823576701835399427"}}},"execution_count":9,"outputs":[]},{"cell_type":"code","source":["parser = argparse.ArgumentParser()\n","\n","arguments = ['--task_name=dataset_single',\n","            '--data_dir=data/dataset/single',\n","            '--vocab_file=ru_conversational_cased_L-12_H-768_A-12/vocab.txt',\n","            '--bert_config_file=ru_conversational_cased_L-12_H-768_A-12/bert_config.json',\n","            '--output_dir=results/dataset/single',\n","            '--init_checkpoint=ru_conversational_cased_L-12_H-768_A-12/pytorch_model.bin',\n","            '--max_seq_len=512']\n","\n","args = parser.parse_args(arguments)\n","\n","if args.local_rank == -1 or args.no_cuda:\n","    device = torch.device(\"cuda\" if torch.cuda.is_available() and not args.no_cuda else \"cpu\")\n","    n_gpu = torch.cuda.device_count()\n","else:\n","    device = torch.device(\"cuda\", args.local_rank)\n","    n_gpu = 1\n","    torch.distributed.init_process_group(backend='nccl')\n","\n","logger.info(\"device %s n_gpu %d distributed training %r\", device, n_gpu, bool(args.local_rank != -1))\n","\n","if args.accumulate_gradients < 1:\n","    raise ValueError(\"Invalid accumulate_gradients parameter: {}, should be >= 1\".format(args.accumulate_gradients))\n","\n","args.train_batch_size = int(args.train_batch_size / args.accumulate_gradients)\n","\n","random.seed(args.seed)\n","np.random.seed(args.seed)\n","torch.manual_seed(args.seed)\n","if n_gpu > 0:\n","    torch.cuda.manual_seed_all(args.seed)\n","\n","bert_config = BertConfig.from_json_file(args.bert_config_file)\n","\n","if args.max_seq_length > bert_config.max_position_embeddings:\n","    raise ValueError(\n","        \"Cannot use sequence length {} because the BERT model was only trained up to sequence length {}\".format(\n","            args.max_seq_length, bert_config.max_position_embeddings))\n","\n","if os.path.exists(args.output_dir) and os.listdir(args.output_dir):\n","    raise ValueError(\"Output directory ({}) already exists and is not empty.\".format(args.output_dir))\n","os.makedirs(args.output_dir, exist_ok=True)\n","\n","# prepare dataloader\n","processors = {\n","    \"dataset_single\": Dataset_single_Processor\n","}\n","\n","processor = processors[args.task_name]()\n","label_list = processor.get_labels()\n","\n","tokenizer = FullTokenizer(vocab_file=args.vocab_file, do_lower_case=args.do_lower_case)\n","\n","# training set\n","train_examples = None\n","num_train_steps = None\n","train_examples = processor.get_train_examples(args.data_dir)\n","num_train_steps = int(len(train_examples) / args.train_batch_size * args.num_train_epochs)\n","\n","train_features = convert_examples_to_features(train_examples, label_list, args.max_seq_length, tokenizer)\n","\n","logger.info(\"***** Running training *****\")\n","logger.info(\"  Num examples = %d\", len(train_examples))\n","logger.info(\"  Batch size = %d\", args.train_batch_size)\n","logger.info(\"  Num steps = %d\", num_train_steps)\n","\n","all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long)\n","all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long)\n","all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long)\n","all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long)\n","train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)\n","\n","if args.local_rank == -1:\n","    train_sampler = RandomSampler(train_data)\n","else:\n","    train_sampler = DistributedSampler(train_data)\n","\n","train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size)\n","\n","# test set\n","if args.eval_test:\n","    test_examples = processor.get_test_examples(args.data_dir)\n","    test_features = convert_examples_to_features(test_examples, label_list, args.max_seq_length, tokenizer)\n","\n","    all_input_ids = torch.tensor([f.input_ids for f in test_features], dtype=torch.long)\n","    all_input_mask = torch.tensor([f.input_mask for f in test_features], dtype=torch.long)\n","    all_segment_ids = torch.tensor([f.segment_ids for f in test_features], dtype=torch.long)\n","    all_label_ids = torch.tensor([f.label_id for f in test_features], dtype=torch.long)\n","\n","    test_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)\n","    test_dataloader = DataLoader(test_data, batch_size=args.eval_batch_size, shuffle=False)\n","\n","# model and optimizer\n","model = BertForSequenceClassification(bert_config, len(label_list))\n","\n","if args.init_checkpoint is not None:\n","    model.bert.load_state_dict(torch.load(args.init_checkpoint, map_location='cpu'))\n","model.to(device)\n","\n","if args.local_rank != -1:\n","    model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank],\n","                                                      output_device=args.local_rank)\n","elif n_gpu > 1:\n","    model = torch.nn.DataParallel(model)\n","\n","no_decay = ['bias', 'gamma', 'beta']\n","optimizer_parameters = [\n","    {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],\n","      'weight_decay_rate': 0.01},\n","    {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],\n","      'weight_decay_rate': 0.0}\n","]\n","\n","optimizer = BERTAdam(optimizer_parameters,\n","                      lr=args.learning_rate,\n","                      warmup=args.warmup_proportion,\n","                      t_total=num_train_steps)\n","\n","# train\n","output_log_file = os.path.join(args.output_dir, \"log.txt\")\n","print(\"output_log_file=\", output_log_file)\n","with open(output_log_file, \"w\") as writer:\n","    if args.eval_test:\n","        writer.write(\"epoch\\tglobal_step\\tloss\\ttest_loss\\ttest_accuracy\\n\")\n","    else:\n","        writer.write(\"epoch\\tglobal_step\\tloss\\n\")\n","\n","global_step = 0\n","epoch = 0\n","for _ in trange(int(args.num_train_epochs), desc=\"Epoch\"):\n","    epoch += 1\n","    model.train()\n","    tr_loss = 0\n","    nb_tr_examples, nb_tr_steps = 0, 0\n","    for step, batch in enumerate(tqdm(train_dataloader, desc=\"Iteration\")):\n","        batch = tuple(t.to(device) for t in batch)\n","        input_ids, input_mask, segment_ids, label_ids = batch\n","        loss, _ = model(input_ids, segment_ids, input_mask, label_ids)\n","        if n_gpu > 1:\n","            loss = loss.mean()  # mean() to average on multi-gpu.\n","        if args.gradient_accumulation_steps > 1:\n","            loss = loss / args.gradient_accumulation_steps\n","        loss.backward()\n","        tr_loss += loss.item()\n","        nb_tr_examples += input_ids.size(0)\n","        nb_tr_steps += 1\n","        if (step + 1) % args.gradient_accumulation_steps == 0:\n","            optimizer.step()  # We have accumulated enought gradients\n","            model.zero_grad()\n","            global_step += 1\n","\n","    # eval_test\n","    if args.eval_test:\n","        model.eval()\n","        test_loss, test_accuracy = 0, 0\n","        nb_test_steps, nb_test_examples = 0, 0\n","        with open(os.path.join(args.output_dir, \"test_ep_\" + str(epoch) + \".txt\"), \"w\") as f_test:\n","            for input_ids, input_mask, segment_ids, label_ids in test_dataloader:\n","                input_ids = input_ids.to(device)\n","                input_mask = input_mask.to(device)\n","                segment_ids = segment_ids.to(device)\n","                label_ids = label_ids.to(device)\n","\n","                with torch.no_grad():\n","                    tmp_test_loss, logits = model(input_ids, segment_ids, input_mask, label_ids)\n","\n","                logits = F.softmax(logits, dim=-1)\n","                logits = logits.detach().cpu().numpy()\n","                label_ids = label_ids.to('cpu').numpy()\n","                outputs = np.argmax(logits, axis=1)\n","                for output_i in range(len(outputs)):\n","                    f_test.write(str(outputs[output_i]))\n","                    for ou in logits[output_i]:\n","                        f_test.write(\" \" + str(ou))\n","                    f_test.write(\"\\n\")\n","                tmp_test_accuracy = np.sum(outputs == label_ids)\n","\n","                test_loss += tmp_test_loss.mean().item()\n","                test_accuracy += tmp_test_accuracy\n","\n","                nb_test_examples += input_ids.size(0)\n","                nb_test_steps += 1\n","\n","        test_loss = test_loss / nb_test_steps\n","        test_accuracy = test_accuracy / nb_test_examples\n","\n","    result = collections.OrderedDict()\n","    if args.eval_test:\n","        result = {'epoch': epoch,\n","                  'global_step': global_step,\n","                  'loss': tr_loss / nb_tr_steps,\n","                  'test_loss': test_loss,\n","                  'test_accuracy': test_accuracy}\n","    else:\n","        result = {'epoch': epoch,\n","                  'global_step': global_step,\n","                  'loss': tr_loss / nb_tr_steps}\n","\n","    logger.info(\"***** Eval results *****\")\n","    with open(output_log_file, \"a+\") as writer:\n","        for key in result.keys():\n","            logger.info(\"  %s = %s\\n\", key, str(result[key]))\n","            writer.write(\"%s\\t\" % (str(result[key])))\n","        writer.write(\"\\n\")\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":168},"id":"CoaNXHrevxBh","executionInfo":{"status":"error","timestamp":1674396588927,"user_tz":-180,"elapsed":241,"user":{"displayName":"Anton Golubev","userId":"02823576701835399427"}},"outputId":"e7e94bdf-f7f4-413b-fa04-5a1e0632665b"},"execution_count":12,"outputs":[{"output_type":"stream","name":"stderr","text":["usage: ipykernel_launcher.py [-h]\n","ipykernel_launcher.py: error: unrecognized arguments: --task_name=dataset_single --data_dir=data/dataset/single --vocab_file=ru_conversational_cased_L-12_H-768_A-12/vocab.txt --bert_config_file=ru_conversational_cased_L-12_H-768_A-12/bert_config.json --output_dir=results/dataset/single --init_checkpoint=ru_conversational_cased_L-12_H-768_A-12/pytorch_model.bin --max_seq_len=512\n"]},{"output_type":"error","ename":"SystemExit","evalue":"ignored","traceback":["An exception has occurred, use %tb to see the full traceback.\n","\u001b[0;31mSystemExit\u001b[0m\u001b[0;31m:\u001b[0m 2\n"]}]},{"cell_type":"code","source":[],"metadata":{"id":"0vyhvaD_zokD"},"execution_count":null,"outputs":[]}]}