InductiveAttentionModels.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "41974af1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "from transformers import GPT2LMHeadModel, GPT2Tokenizer, AdamW, get_linear_schedule_with_warmup, GPT2Config\n",
    "from transformers import GPT2Config, GPT2Tokenizer, GPT2LMHeadModel,GPT2Model, AdamW, get_linear_schedule_with_warmup, BertModel, BertTokenizer\n",
    "from transformers import modeling_outputs, modeling_utils\n",
    "from transformers.modeling_outputs import CausalLMOutputWithCrossAttentions\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ccb9b0bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MLP(nn.Module):\n",
    "    def __init__(self, n_state, config):  # in MLP: n_state=3072 (4 * n_embd)\n",
    "        super(MLP, self).__init__()\n",
    "        nx = config.n_embd\n",
    "        self.c_fc = modeling_utils.Conv1D(n_state, nx)\n",
    "        self.c_proj = modeling_utils.Conv1D(nx, n_state)\n",
    "        self.act = gelu\n",
    "        self.dropout = nn.Dropout(config.resid_pdrop)\n",
    "\n",
    "    def forward(self, x):\n",
    "        h = self.act(self.c_fc(x))\n",
    "        h2 = self.c_proj(h)\n",
    "        return self.dropout(h2)\n",
    "def gelu(x):\n",
    "    return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))\n",
    "\n",
    "class BlockIA(nn.Module):\n",
    "    def __init__(self, n_ctx, config, scale=False):\n",
    "        super().__init__()\n",
    "        hidden_size = config.n_embd\n",
    "        inner_dim = config.n_inner if config.n_inner is not None else 4 * hidden_size\n",
    "        self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)\n",
    "        self.attn = AttentionIA(hidden_size, n_ctx, config, scale)\n",
    "        self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)\n",
    "        if config.add_cross_attention:\n",
    "            self.crossattention = AttentionIA(hidden_size, n_ctx, config, scale, is_cross_attention=True)\n",
    "            self.ln_cross_attn = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)\n",
    "        self.mlp = MLP(inner_dim, config)\n",
    "\n",
    "    def forward(\n",
    "        self,\n",
    "        hidden_states,\n",
    "        layer_past=None,\n",
    "        attention_mask=None,\n",
    "        head_mask=None,\n",
    "        encoder_hidden_states=None,\n",
    "        encoder_attention_mask=None,\n",
    "        use_cache=False,\n",
    "        output_attentions=False,\n",
    "        inductive_attention_mask=None,\n",
    "    ):\n",
    "        attn_outputs = self.attn(\n",
    "            self.ln_1(hidden_states),\n",
    "            layer_past=layer_past,\n",
    "            attention_mask=attention_mask,\n",
    "            head_mask=head_mask,\n",
    "            use_cache=use_cache,\n",
    "            output_attentions=output_attentions,\n",
    "            inductive_attention_mask=inductive_attention_mask,\n",
    "        )\n",
    "        # import pdb;pdb.set_trace()\n",
    "        attn_output = attn_outputs[0]  # output_attn: a, present, (attentions)\n",
    "        outputs = attn_outputs[1:]\n",
    "        # residual connection\n",
    "        hidden_states = attn_output + hidden_states\n",
    "\n",
    "        if encoder_hidden_states is not None:\n",
    "            # add one self-attention block for cross-attention\n",
    "            assert hasattr(\n",
    "                self, \"crossattention\"\n",
    "            ), f\"If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`\"\n",
    "            cross_attn_outputs = self.crossattention(\n",
    "                self.ln_cross_attn(hidden_states),\n",
    "                attention_mask=attention_mask,\n",
    "                head_mask=head_mask,\n",
    "                encoder_hidden_states=encoder_hidden_states,\n",
    "                encoder_attention_mask=encoder_attention_mask,\n",
    "                output_attentions=output_attentions,\n",
    "            )\n",
    "            attn_output = cross_attn_outputs[0]\n",
    "            # residual connection\n",
    "            hidden_states = hidden_states + attn_output\n",
    "            outputs = outputs + cross_attn_outputs[2:]  # add cross attentions if we output attention weights\n",
    "\n",
    "        feed_forward_hidden_states = self.mlp(self.ln_2(hidden_states))\n",
    "        # residual connection\n",
    "        hidden_states = hidden_states + feed_forward_hidden_states\n",
    "\n",
    "        if use_cache:\n",
    "            outputs = (hidden_states,) + outputs\n",
    "        else:\n",
    "            outputs = (hidden_states,) + outputs[1:]\n",
    "\n",
    "        return outputs  # hidden_states, present, (attentions, cross_attentions)\n",
    "\n",
    "class AttentionIA(nn.Module):\n",
    "    def __init__(self, nx, n_ctx, config, scale=False, is_cross_attention=False):\n",
    "        super().__init__()\n",
    "\n",
    "        n_state = nx  # in Attention: n_state=768 (nx=n_embd)\n",
    "        # [switch nx => n_state from Block to Attention to keep identical to TF implem]\n",
    "        assert n_state % config.n_head == 0\n",
    "        self.register_buffer(\n",
    "            \"bias\", torch.tril(torch.ones((n_ctx, n_ctx), dtype=torch.uint8)).view(1, 1, n_ctx, n_ctx)\n",
    "        )\n",
    "        self.register_buffer(\"masked_bias\", torch.tensor(-1e4))\n",
    "        self.n_head = config.n_head\n",
    "        self.split_size = n_state\n",
    "        self.scale = scale\n",
    "        self.is_cross_attention = is_cross_attention\n",
    "        if self.is_cross_attention:\n",
    "            self.c_attn = modeling_utils.Conv1D(2 * n_state, nx)\n",
    "            self.q_attn = modeling_utils.Conv1D(n_state, nx)\n",
    "        else:\n",
    "            self.c_attn = modeling_utils.Conv1D(3 * n_state, nx)\n",
    "        self.c_proj = modeling_utils.Conv1D(n_state, nx)\n",
    "        self.attn_dropout = nn.Dropout(config.attn_pdrop)\n",
    "        self.resid_dropout = nn.Dropout(config.resid_pdrop)\n",
    "        self.pruned_heads = set()\n",
    "\n",
    "    def prune_heads(self, heads):\n",
    "        if len(heads) == 0:\n",
    "            return\n",
    "        heads, index = find_pruneable_heads_and_indices(\n",
    "            heads, self.n_head, self.split_size // self.n_head, self.pruned_heads\n",
    "        )\n",
    "        index_attn = torch.cat([index, index + self.split_size, index + (2 * self.split_size)])\n",
    "\n",
    "        # Prune conv1d layers\n",
    "        self.c_attn = prune_conv1d_layer(self.c_attn, index_attn, dim=1)\n",
    "        self.c_proj = prune_conv1d_layer(self.c_proj, index, dim=0)\n",
    "\n",
    "        # Update hyper params\n",
    "        self.split_size = (self.split_size // self.n_head) * (self.n_head - len(heads))\n",
    "        self.n_head = self.n_head - len(heads)\n",
    "        self.pruned_heads = self.pruned_heads.union(heads)\n",
    "\n",
    "    def _attn(self, q, k, v, attention_mask=None, head_mask=None, output_attentions=False, inductive_attention_mask=None):\n",
    "        # import pdb;pdb.set_trace();\n",
    "        w = torch.matmul(q, k)\n",
    "        if self.scale:\n",
    "            w = w / (float(v.size(-1)) ** 0.5)\n",
    "        nd, ns = w.size(-2), w.size(-1)\n",
    "\n",
    "        if not self.is_cross_attention:\n",
    "            # if only \"normal\" attention layer implements causal mask\n",
    "            mask = self.bias[:, :, ns - nd : ns, :ns] #[1,1,len,len]\n",
    "            if inductive_attention_mask != None:\n",
    "                mask = mask + inductive_attention_mask\n",
    "            w = torch.where(mask.bool(), w, self.masked_bias.to(w.dtype))\n",
    "\n",
    "        if attention_mask is not None:\n",
    "            # Apply the attention mask\n",
    "            w = w + attention_mask\n",
    "\n",
    "        w = nn.Softmax(dim=-1)(w)\n",
    "        w = self.attn_dropout(w)\n",
    "\n",
    "        # Mask heads if we want to\n",
    "        if head_mask is not None:\n",
    "            w = w * head_mask\n",
    "\n",
    "        outputs = (torch.matmul(w, v),)\n",
    "        if output_attentions:\n",
    "            outputs += (w,)\n",
    "        return outputs\n",
    "\n",
    "    def merge_heads(self, x):\n",
    "        x = x.permute(0, 2, 1, 3).contiguous()\n",
    "        new_x_shape = x.size()[:-2] + (x.size(-2) * x.size(-1),)\n",
    "        return x.view(*new_x_shape)  # in Tensorflow implem: fct merge_states\n",
    "\n",
    "    def split_heads(self, x, k=False):\n",
    "        new_x_shape = x.size()[:-1] + (self.n_head, x.size(-1) // self.n_head)\n",
    "        x = x.view(*new_x_shape)  # in Tensorflow implem: fct split_states\n",
    "        if k:\n",
    "            return x.permute(0, 2, 3, 1)  # (batch, head, head_features, seq_length)\n",
    "        else:\n",
    "            return x.permute(0, 2, 1, 3)  # (batch, head, seq_length, head_features)\n",
    "\n",
    "    def forward(\n",
    "        self,\n",
    "        hidden_states,\n",
    "        layer_past=None,\n",
    "        attention_mask=None,\n",
    "        head_mask=None,\n",
    "        encoder_hidden_states=None,\n",
    "        encoder_attention_mask=None,\n",
    "        use_cache=False,\n",
    "        output_attentions=False,\n",
    "        inductive_attention_mask=None,\n",
    "    ):\n",
    "        if encoder_hidden_states is not None:\n",
    "            assert hasattr(\n",
    "                self, \"q_attn\"\n",
    "            ), \"If class is used as cross attention, the weights `q_attn` have to be defined. Please make sure to instantiate class with `Attention(..., is_cross_attention=True)`.\"\n",
    "            query = self.q_attn(hidden_states)\n",
    "            key, value = self.c_attn(encoder_hidden_states).split(self.split_size, dim=2)\n",
    "            attention_mask = encoder_attention_mask\n",
    "        else:\n",
    "            query, key, value = self.c_attn(hidden_states).split(self.split_size, dim=2)\n",
    "\n",
    "        query = self.split_heads(query)\n",
    "        key = self.split_heads(key, k=True)\n",
    "        value = self.split_heads(value)\n",
    "        if layer_past is not None:\n",
    "            past_key, past_value = layer_past[0].transpose(-2, -1), layer_past[1]  # transpose back cf below\n",
    "            key = torch.cat((past_key, key), dim=-1)\n",
    "            value = torch.cat((past_value, value), dim=-2)\n",
    "\n",
    "        if use_cache is True:\n",
    "            present = (key.transpose(-2, -1), value)  # transpose to have same shapes\n",
    "        else:\n",
    "            present = None\n",
    "\n",
    "        attn_outputs = self._attn(query, key, value, attention_mask, head_mask, output_attentions, inductive_attention_mask)\n",
    "        a = attn_outputs[0]\n",
    "\n",
    "        a = self.merge_heads(a)\n",
    "        a = self.c_proj(a)\n",
    "        a = self.resid_dropout(a)\n",
    "\n",
    "        return (a, present) + attn_outputs[1:]  # a, present, (attentions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "832af02e",
   "metadata": {},
   "outputs": [],
   "source": [
    "class GPT2InductiveAttention(GPT2Model):\n",
    "    def __init__(self, config):\n",
    "        super(GPT2InductiveAttention, self).__init__(config)\n",
    "        self.h = nn.ModuleList([BlockIA(config.n_ctx, config, scale=True) for _ in range(config.n_layer)])\n",
    "    def forward(\n",
    "        self,\n",
    "        input_ids=None,\n",
    "        past_key_values=None,\n",
    "        attention_mask=None,\n",
    "        inductive_attention_mask=None,\n",
    "        token_type_ids=None,\n",
    "        position_ids=None,\n",
    "        head_mask=None,\n",
    "        inputs_embeds=None,\n",
    "        encoder_hidden_states=None,\n",
    "        encoder_attention_mask=None,\n",
    "        use_cache=None,\n",
    "        output_attentions=None,\n",
    "        output_hidden_states=None,\n",
    "        return_dict=None,\n",
    "    ):\n",
    "        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions\n",
    "        output_hidden_states = (\n",
    "            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states\n",
    "        )\n",
    "        use_cache = use_cache if use_cache is not None else self.config.use_cache\n",
    "        return_dict = return_dict if return_dict is not None else self.config.use_return_dict\n",
    "\n",
    "        if input_ids is not None and inputs_embeds is not None:\n",
    "            raise ValueError(\"You cannot specify both input_ids and inputs_embeds at the same time\")\n",
    "        elif input_ids is not None:\n",
    "            input_shape = input_ids.size()\n",
    "            input_ids = input_ids.view(-1, input_shape[-1])\n",
    "            batch_size = input_ids.shape[0]\n",
    "        elif inputs_embeds is not None:\n",
    "            input_shape = inputs_embeds.size()[:-1]\n",
    "            batch_size = inputs_embeds.shape[0]\n",
    "        else:\n",
    "            raise ValueError(\"You have to specify either input_ids or inputs_embeds\")\n",
    "\n",
    "        if token_type_ids is not None:\n",
    "            token_type_ids = token_type_ids.view(-1, input_shape[-1])\n",
    "        if position_ids is not None:\n",
    "            position_ids = position_ids.view(-1, input_shape[-1])\n",
    "\n",
    "        if past_key_values is None:\n",
    "            past_length = 0\n",
    "            past_key_values = tuple([None] * len(self.h))\n",
    "        else:\n",
    "            past_length = past_key_values[0][0].size(-2)\n",
    "        if position_ids is None:\n",
    "            device = input_ids.device if input_ids is not None else inputs_embeds.device\n",
    "            position_ids = torch.arange(past_length, input_shape[-1] + past_length, dtype=torch.long, device=device)\n",
    "            position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1])\n",
    "        # Attention mask.\n",
    "        if attention_mask is not None:\n",
    "            assert batch_size > 0, \"batch_size has to be defined and > 0\"\n",
    "            attention_mask = attention_mask.view(batch_size, -1)\n",
    "            # We create a 3D attention mask from a 2D tensor mask.\n",
    "            # Sizes are [batch_size, 1, 1, to_seq_length]\n",
    "            # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]\n",
    "            # this attention mask is more simple than the triangular masking of causal attention\n",
    "            # used in OpenAI GPT, we just need to prepare the broadcast dimension here.\n",
    "            attention_mask = attention_mask[:, None, None, :]\n",
    "\n",
    "            # Since attention_mask is 1.0 for positions we want to attend and 0.0 for\n",
    "            # masked positions, this operation will create a tensor which is 0.0 for\n",
    "            # positions we want to attend and -10000.0 for masked positions.\n",
    "            # Since we are adding it to the raw scores before the softmax, this is\n",
    "            # effectively the same as removing these entirely.\n",
    "            attention_mask = attention_mask.to(dtype=self.dtype)  # fp16 compatibility\n",
    "            attention_mask = (1.0 - attention_mask) * -10000.0\n",
    "\n",
    "        # If a 2D ou 3D attention mask is provided for the cross-attention\n",
    "        # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]\n",
    "        if self.config.add_cross_attention and encoder_hidden_states is not None:\n",
    "            encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size()\n",
    "            encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)\n",
    "            if encoder_attention_mask is None:\n",
    "                encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device)\n",
    "            encoder_attention_mask = self.invert_attention_mask(encoder_attention_mask)\n",
    "        else:\n",
    "            encoder_attention_mask = None\n",
    "\n",
    "        # Prepare head mask if needed\n",
    "        # 1.0 in head_mask indicate we keep the head\n",
    "        # attention_probs has shape bsz x n_heads x N x N\n",
    "        # head_mask has shape n_layer x batch x n_heads x N x N\n",
    "        head_mask = self.get_head_mask(head_mask, self.config.n_layer)\n",
    "\n",
    "        if inputs_embeds is None:\n",
    "            inputs_embeds = self.wte(input_ids)\n",
    "        position_embeds = self.wpe(position_ids)\n",
    "        hidden_states = inputs_embeds + position_embeds\n",
    "\n",
    "        if token_type_ids is not None:\n",
    "            token_type_embeds = self.wte(token_type_ids)\n",
    "            hidden_states = hidden_states + token_type_embeds\n",
    "\n",
    "        hidden_states = self.drop(hidden_states)\n",
    "\n",
    "        output_shape = input_shape + (hidden_states.size(-1),)\n",
    "\n",
    "        presents = () if use_cache else None\n",
    "        all_self_attentions = () if output_attentions else None\n",
    "        all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None\n",
    "        all_hidden_states = () if output_hidden_states else None\n",
    "        for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)):\n",
    "            # Model parallel\n",
    "            if self.model_parallel:\n",
    "                torch.cuda.set_device(hidden_states.device)\n",
    "                # Ensure layer_past is on same device as hidden_states (might not be correct)\n",
    "                if layer_past is not None:\n",
    "                    layer_past = tuple(past_state.to(hidden_states.device) for past_state in layer_past)\n",
    "                # Ensure that attention_mask is always on the same device as hidden_states\n",
    "                if attention_mask is not None:\n",
    "                    attention_mask = attention_mask.to(hidden_states.device)\n",
    "                if isinstance(head_mask, torch.Tensor):\n",
    "                    head_mask = head_mask.to(hidden_states.device)\n",
    "            if output_hidden_states:\n",
    "                all_hidden_states = all_hidden_states + (hidden_states,)\n",
    "\n",
    "            \n",
    "            if getattr(self.config, \"gradient_checkpointing\", False) and self.training:\n",
    "\n",
    "                if use_cache:\n",
    "                    logger.warn(\n",
    "                        \"`use_cache=True` is incompatible with `config.gradient_checkpointing=True`. Setting \"\n",
    "                        \"`use_cache=False`...\"\n",
    "                    )\n",
    "                    use_cache = False\n",
    "\n",
    "                def create_custom_forward(module):\n",
    "                    def custom_forward(*inputs):\n",
    "                        # None for past_key_value\n",
    "                        return module(*inputs, use_cache, output_attentions)\n",
    "\n",
    "                    return custom_forward\n",
    "\n",
    "                outputs = torch.utils.checkpoint.checkpoint(\n",
    "                    create_custom_forward(block),\n",
    "                    hidden_states,\n",
    "                    None,\n",
    "                    attention_mask,\n",
    "                    head_mask[i],\n",
    "                    encoder_hidden_states,\n",
    "                    encoder_attention_mask,\n",
    "                )\n",
    "            else:\n",
    "                outputs = block(\n",
    "                    hidden_states,\n",
    "                    layer_past=layer_past,\n",
    "                    attention_mask=attention_mask,\n",
    "                    head_mask=head_mask[i],\n",
    "                    encoder_hidden_states=encoder_hidden_states,\n",
    "                    encoder_attention_mask=encoder_attention_mask,\n",
    "                    use_cache=use_cache,\n",
    "                    output_attentions=output_attentions,\n",
    "                    inductive_attention_mask=inductive_attention_mask,\n",
    "                )\n",
    "            \n",
    "            hidden_states = outputs[0]\n",
    "            if use_cache is True:\n",
    "                presents = presents + (outputs[1],)\n",
    "\n",
    "            if output_attentions:\n",
    "                all_self_attentions = all_self_attentions + (outputs[2 if use_cache else 1],)\n",
    "                if self.config.add_cross_attention:\n",
    "                    all_cross_attentions = all_cross_attentions + (outputs[3 if use_cache else 2],)\n",
    "\n",
    "            # Model Parallel: If it's the last layer for that device, put things on the next device\n",
    "            if self.model_parallel:\n",
    "                for k, v in self.device_map.items():\n",
    "                    if i == v[-1] and \"cuda:\" + str(k) != self.last_device:\n",
    "                        hidden_states = hidden_states.to(\"cuda:\" + str(k + 1))\n",
    "\n",
    "        hidden_states = self.ln_f(hidden_states)\n",
    "\n",
    "        hidden_states = hidden_states.view(*output_shape)\n",
    "        # Add last hidden state\n",
    "        if output_hidden_states:\n",
    "            all_hidden_states = all_hidden_states + (hidden_states,)\n",
    "\n",
    "        if not return_dict:\n",
    "            return tuple(v for v in [hidden_states, presents, all_hidden_states, all_self_attentions] if v is not None)\n",
    "\n",
    "        return modeling_outputs.BaseModelOutputWithPastAndCrossAttentions(\n",
    "            last_hidden_state=hidden_states,\n",
    "            past_key_values=presents,\n",
    "            hidden_states=all_hidden_states,\n",
    "            attentions=all_self_attentions,\n",
    "            cross_attentions=all_cross_attentions,\n",
    "        )\n",
    "class GPT2InductiveAttentionHeadModel(GPT2LMHeadModel):\n",
    "    def __init__(self, config):\n",
    "        super(GPT2InductiveAttentionHeadModel, self).__init__(config)\n",
    "        self.transformer = GPT2InductiveAttention(config)\n",
    "    \n",
    "    def forward(\n",
    "        self,\n",
    "        input_ids=None,\n",
    "        past_key_values=None,\n",
    "        attention_mask=None,\n",
    "        inductive_attention_mask=None, #[len,len]\n",
    "        token_type_ids=None,\n",
    "        position_ids=None,\n",
    "        head_mask=None,\n",
    "        inputs_embeds=None,\n",
    "        encoder_hidden_states=None,\n",
    "        encoder_attention_mask=None,\n",
    "        labels=None,\n",
    "        use_cache=None,\n",
    "        output_attentions=None,\n",
    "        output_hidden_states=None,\n",
    "        return_dict=None,\n",
    "    ):\n",
    "        r\"\"\"\n",
    "        labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):\n",
    "            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set\n",
    "            ``labels = input_ids`` Indices are selected in ``[-100, 0, ..., config.vocab_size]`` All labels set to\n",
    "            ``-100`` are ignored (masked), the loss is only computed for labels in ``[0, ..., config.vocab_size]``\n",
    "        \"\"\"\n",
    "        return_dict = return_dict if return_dict is not None else self.config.use_return_dict\n",
    "\n",
    "        transformer_outputs = self.transformer(\n",
    "            input_ids,\n",
    "            past_key_values=past_key_values,\n",
    "            attention_mask=attention_mask,\n",
    "            inductive_attention_mask=inductive_attention_mask,\n",
    "            token_type_ids=token_type_ids,\n",
    "            position_ids=position_ids,\n",
    "            head_mask=head_mask,\n",
    "            inputs_embeds=inputs_embeds,\n",
    "            encoder_hidden_states=encoder_hidden_states,\n",
    "            encoder_attention_mask=encoder_attention_mask,\n",
    "            use_cache=use_cache,\n",
    "            output_attentions=output_attentions,\n",
    "            output_hidden_states=output_hidden_states,\n",
    "            return_dict=return_dict,\n",
    "        )\n",
    "        hidden_states = transformer_outputs[0]\n",
    "\n",
    "        # Set device for model parallelism\n",
    "        if self.model_parallel:\n",
    "            torch.cuda.set_device(self.transformer.first_device)\n",
    "            hidden_states = hidden_states.to(self.lm_head.weight.device)\n",
    "\n",
    "        lm_logits = self.lm_head(hidden_states)\n",
    "\n",
    "        loss = None\n",
    "        if labels is not None:\n",
    "            # Shift so that tokens < n predict n\n",
    "            shift_logits = lm_logits[..., :-1, :].contiguous()\n",
    "            shift_labels = labels[..., 1:].contiguous()\n",
    "            # Flatten the tokens\n",
    "            loss_fct = CrossEntropyLoss()\n",
    "            loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))\n",
    "\n",
    "        if not return_dict:\n",
    "            output = (lm_logits,) + transformer_outputs[1:]\n",
    "            return ((loss,) + output) if loss is not None else output\n",
    "\n",
    "        return CausalLMOutputWithCrossAttentions(\n",
    "            loss=loss,\n",
    "            logits=lm_logits,\n",
    "            past_key_values=transformer_outputs.past_key_values,\n",
    "            hidden_states=transformer_outputs.hidden_states,\n",
    "            attentions=transformer_outputs.attentions,\n",
    "            cross_attentions=transformer_outputs.cross_attentions,\n",
    "        )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b7ea5754",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2011098/981292096.py:4: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  tokens = torch.tensor(tokens[\"input_ids\"])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[[ -36.0205,  -36.0670,  -39.6015,  ...,  -45.2210,  -44.6506,\n",
       "           -37.3549],\n",
       "         [-100.2482,  -98.5362, -105.1653,  ..., -111.2344, -112.6463,\n",
       "          -101.2943],\n",
       "         [ -84.4618,  -84.8718,  -87.6387,  ...,  -88.3179,  -92.3162,\n",
       "           -85.0362],\n",
       "         [ -98.3060, -100.1567, -104.9867,  ..., -107.9936, -109.8437,\n",
       "          -103.2899]]], grad_fn=<UnsafeViewBackward>)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = GPT2InductiveAttentionHeadModel.from_pretrained('gpt2')\n",
    "gpt_tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
    "tokens = gpt_tokenizer(\"Hey How are you\", return_tensors=\"pt\")\n",
    "tokens = torch.tensor(tokens[\"input_ids\"])\n",
    "model(tokens).logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "516eaba5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# model.transformer.wpe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "033281d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2011098/2485755930.py:4: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  tokens = model.transformer.wte(torch.tensor(tokens[\"input_ids\"]))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[[ -36.0205,  -36.0670,  -39.6015,  ...,  -45.2210,  -44.6506,\n",
       "           -37.3549],\n",
       "         [-100.2482,  -98.5362, -105.1653,  ..., -111.2344, -112.6463,\n",
       "          -101.2943],\n",
       "         [ -84.4618,  -84.8718,  -87.6387,  ...,  -88.3179,  -92.3162,\n",
       "           -85.0362],\n",
       "         [ -98.3060, -100.1567, -104.9867,  ..., -107.9936, -109.8437,\n",
       "          -103.2899]]], grad_fn=<UnsafeViewBackward>)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = GPT2InductiveAttentionHeadModel.from_pretrained('gpt2')\n",
    "gpt_tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
    "tokens = gpt_tokenizer(\"Hey How are you\", return_tensors=\"pt\")\n",
    "tokens = model.transformer.wte(torch.tensor(tokens[\"input_ids\"]))\n",
    "model(inputs_embeds=tokens, output_hidden_states=True).logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c6949de",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}