multi-peft.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导入必要的库和函数\n",
    "from transformers import AutoModelForSeq2SeqLM, AutoModelForSequenceClassification  # 用于加载和处理序列到序列的语言模型\n",
    "# 导入必要的库\n",
    "from adapters import AdapterTrainer\n",
    "from transformers import TrainingArguments, EvalPrediction\n",
    "import numpy as np\n",
    "from transformers import RobertaConfig\n",
    "from transformers import RobertaTokenizer\n",
    "import torch\n",
    "from transformers import RobertaTokenizer, RobertaForSequenceClassification\n",
    "from adapters import AutoAdapterModel, AdapterArguments, AdapterTrainer, AdapterConfig, ConfigUnion, LoRAConfig, SeqBnConfig, PrefixTuningConfig\n",
    "from datasets import load_dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using the latest cached version of the dataset since rotten_tomatoes couldn't be found on the Hugging Face Hub\n",
      "Found the latest cached dataset configuration 'default' at C:\\Users\\13061\\.cache\\huggingface\\datasets\\rotten_tomatoes\\default\\1.0.0\\c699dc617d02c6738bbcb70f35c1875d20011526 (last modified on Thu Mar  7 11:17:13 2024).\n"
     ]
    }
   ],
   "source": [
    "# 定义一些参数\n",
    "model_name_or_path = \"roberta-base\"\n",
    "task_name = \"rotten_tomatoes\"\n",
    "output_dir = \"out/roberta-base-rotten_tomatoes/\"\n",
    "\n",
    "# 加载数据集\n",
    "# dataset = load_dataset(\"glue\", task_name)\n",
    "dataset = load_dataset(\"rotten_tomatoes\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of RobertaAdapterModel were not initialized from the model checkpoint at roberta-base and are newly initialized: ['heads.default.3.bias', 'roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainable params: 125288025 || all params: 125288025 || trainable%: 100.00\n",
      "trainable params: 2155158 || all params: 126851055 || trainable%: 1.70\n"
     ]
    }
   ],
   "source": [
    "# 添加lora和adapter\n",
    "# 加载预训练的序列到序列语言模型\n",
    "model = AutoAdapterModel.from_pretrained(model_name_or_path)\n",
    "tokenizer = RobertaTokenizer.from_pretrained(\"roberta-base\")\n",
    "\n",
    "print_trainable_parameters(model)\n",
    "\n",
    "# 配置PEFT的参数\n",
    "lora_config = LoRAConfig(\n",
    "    r=8,  # 设置LoRA的rank\n",
    "    alpha=32,  # LoRA的alpha值，决定参数增加的数量\n",
    "    dropout=0.1,  # LoRA层的dropout比例\n",
    "    leave_out=[6, 7, 8, 9, 10, 11],  # 指定需要转换的层 #important\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "bn_config = SeqBnConfig(\n",
    "    reduction_factor=5,  # 设置瓶颈维度\n",
    "    leave_out=[6, 7, 8, 9, 10, 11]  # 指定需要转换的层 #important\n",
    ")\n",
    "\n",
    "config_list=[lora_config, bn_config]\n",
    "peft_config=ConfigUnion(*[config_list[i] for i in range(len(config_list))])\n",
    "\n",
    "\n",
    "model.add_adapter(\"rotten_tomatoes\",peft_config)\n",
    "model.train_adapter(\"rotten_tomatoes\")\n",
    "model.set_active_adapters(\"rotten_tomatoes\")\n",
    "\n",
    "\n",
    "\n",
    "print_trainable_parameters(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of RobertaAdapterModel were not initialized from the model checkpoint at roberta-base and are newly initialized: ['heads.default.3.bias', 'roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainable params: 125288025 || all params: 125288025 || trainable%: 100.00\n",
      "trainable params: 739584 || all params: 125435481 || trainable%: 0.59\n"
     ]
    }
   ],
   "source": [
    "# 仅添加一个lora\n",
    "# 加载预训练的序列到序列语言模型\n",
    "model = AutoAdapterModel.from_pretrained(model_name_or_path)\n",
    "tokenizer = RobertaTokenizer.from_pretrained(\"roberta-base\")\n",
    "\n",
    "print_trainable_parameters(model)\n",
    "\n",
    "# 配置PEFT的参数\n",
    "lora_config = LoRAConfig(\n",
    "    r=8,  # 设置LoRA的rank\n",
    "    alpha=32,  # LoRA的alpha值，决定参数增加的数量\n",
    "    dropout=0.1,  # LoRA层的dropout比例\n",
    "    leave_out=[6, 7, 8, 9, 10, 11],  # 指定需要转换的层 #important\n",
    ")\n",
    "\n",
    "\n",
    "# bn_config = SeqBnConfig(\n",
    "\n",
    "#     reduction_factor=0.2,  # 设置LoRA的rank\n",
    "\n",
    "#     leave_out=[0, 1, 2, 3, 4, 5]  # 指定需要转换的层 #important\n",
    "\n",
    "# )\n",
    "\n",
    "# config_list=[lora_config, bn_config]\n",
    "# peft_config=ConfigUnion(*[config_list[i] for i in range(len(config_list))])\n",
    "\n",
    "\n",
    "model.add_adapter(\"rotten_tomatoes\", lora_config)\n",
    "model.train_adapter(\"rotten_tomatoes\")\n",
    "model.set_active_adapters(\"rotten_tomatoes\")\n",
    "\n",
    "\n",
    "print_trainable_parameters(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "roberta\n",
      "roberta.embeddings\n",
      "roberta.embeddings.word_embeddings\n",
      "roberta.embeddings.position_embeddings\n",
      "roberta.embeddings.token_type_embeddings\n",
      "roberta.embeddings.LayerNorm\n",
      "roberta.embeddings.dropout\n",
      "roberta.encoder\n",
      "roberta.encoder.layer\n",
      "roberta.encoder.layer.0\n",
      "roberta.encoder.layer.0.attention\n",
      "roberta.encoder.layer.0.attention.self\n",
      "roberta.encoder.layer.0.attention.self.query\n",
      "roberta.encoder.layer.0.attention.self.query.loras\n",
      "roberta.encoder.layer.0.attention.self.query.loras.rotten_tomatoes\n",
      "roberta.encoder.layer.0.attention.self.query.loras.rotten_tomatoes.lora_dropout\n",
      "roberta.encoder.layer.0.attention.self.key\n",
      "roberta.encoder.layer.0.attention.self.key.loras\n",
      "roberta.encoder.layer.0.attention.self.value\n",
      "roberta.encoder.layer.0.attention.self.value.loras\n",
      "roberta.encoder.layer.0.attention.self.value.loras.rotten_tomatoes\n",
      "roberta.encoder.layer.0.attention.self.value.loras.rotten_tomatoes.lora_dropout\n",
      "roberta.encoder.layer.0.attention.self.dropout\n",
      "roberta.encoder.layer.0.attention.self.prefix_tuning\n",
      "roberta.encoder.layer.0.attention.self.prefix_tuning.prefix_gates\n",
      "roberta.encoder.layer.0.attention.self.prefix_tuning.pool\n",
      "roberta.encoder.layer.0.attention.self.prefix_tuning.pool.prefix_tunings\n",
      "roberta.encoder.layer.0.attention.output\n",
      "roberta.encoder.layer.0.attention.output.dense\n",
      "roberta.encoder.layer.0.attention.output.LayerNorm\n",
      "roberta.encoder.layer.0.attention.output.dropout\n",
      "roberta.encoder.layer.0.attention.output.adapters\n",
      "roberta.encoder.layer.0.attention.output.adapter_fusion_layer\n",
      "roberta.encoder.layer.0.intermediate\n",
      "roberta.encoder.layer.0.intermediate.dense\n",
      "roberta.encoder.layer.0.intermediate.dense.loras\n",
      "roberta.encoder.layer.0.intermediate.intermediate_act_fn\n",
      "roberta.encoder.layer.0.output\n",
      "roberta.encoder.layer.0.output.dense\n",
      "roberta.encoder.layer.0.output.dense.loras\n",
      "roberta.encoder.layer.0.output.LayerNorm\n",
      "roberta.encoder.layer.0.output.dropout\n",
      "roberta.encoder.layer.0.output.adapters\n",
      "roberta.encoder.layer.0.output.adapters.rotten_tomatoes\n",
      "roberta.encoder.layer.0.output.adapters.rotten_tomatoes.non_linearity\n",
      "roberta.encoder.layer.0.output.adapters.rotten_tomatoes.non_linearity.f\n",
      "roberta.encoder.layer.0.output.adapters.rotten_tomatoes.adapter_down\n",
      "roberta.encoder.layer.0.output.adapters.rotten_tomatoes.adapter_down.0\n",
      "roberta.encoder.layer.0.output.adapters.rotten_tomatoes.adapter_up\n",
      "roberta.encoder.layer.0.output.adapter_fusion_layer\n",
      "roberta.encoder.layer.1\n",
      "roberta.encoder.layer.1.attention\n",
      "roberta.encoder.layer.1.attention.self\n",
      "roberta.encoder.layer.1.attention.self.query\n",
      "roberta.encoder.layer.1.attention.self.query.loras\n",
      "roberta.encoder.layer.1.attention.self.query.loras.rotten_tomatoes\n",
      "roberta.encoder.layer.1.attention.self.query.loras.rotten_tomatoes.lora_dropout\n",
      "roberta.encoder.layer.1.attention.self.key\n",
      "roberta.encoder.layer.1.attention.self.key.loras\n",
      "roberta.encoder.layer.1.attention.self.value\n",
      "roberta.encoder.layer.1.attention.self.value.loras\n",
      "roberta.encoder.layer.1.attention.self.value.loras.rotten_tomatoes\n",
      "roberta.encoder.layer.1.attention.self.value.loras.rotten_tomatoes.lora_dropout\n",
      "roberta.encoder.layer.1.attention.self.dropout\n",
      "roberta.encoder.layer.1.attention.self.prefix_tuning\n",
      "roberta.encoder.layer.1.attention.self.prefix_tuning.prefix_gates\n",
      "roberta.encoder.layer.1.attention.output\n",
      "roberta.encoder.layer.1.attention.output.dense\n",
      "roberta.encoder.layer.1.attention.output.LayerNorm\n",
      "roberta.encoder.layer.1.attention.output.dropout\n",
      "roberta.encoder.layer.1.attention.output.adapters\n",
      "roberta.encoder.layer.1.attention.output.adapter_fusion_layer\n",
      "roberta.encoder.layer.1.intermediate\n",
      "roberta.encoder.layer.1.intermediate.dense\n",
      "roberta.encoder.layer.1.intermediate.dense.loras\n",
      "roberta.encoder.layer.1.intermediate.intermediate_act_fn\n",
      "roberta.encoder.layer.1.output\n",
      "roberta.encoder.layer.1.output.dense\n",
      "roberta.encoder.layer.1.output.dense.loras\n",
      "roberta.encoder.layer.1.output.LayerNorm\n",
      "roberta.encoder.layer.1.output.dropout\n",
      "roberta.encoder.layer.1.output.adapters\n",
      "roberta.encoder.layer.1.output.adapters.rotten_tomatoes\n",
      "roberta.encoder.layer.1.output.adapters.rotten_tomatoes.non_linearity\n",
      "roberta.encoder.layer.1.output.adapters.rotten_tomatoes.non_linearity.f\n",
      "roberta.encoder.layer.1.output.adapters.rotten_tomatoes.adapter_down\n",
      "roberta.encoder.layer.1.output.adapters.rotten_tomatoes.adapter_down.0\n",
      "roberta.encoder.layer.1.output.adapters.rotten_tomatoes.adapter_up\n",
      "roberta.encoder.layer.1.output.adapter_fusion_layer\n",
      "roberta.encoder.layer.2\n",
      "roberta.encoder.layer.2.attention\n",
      "roberta.encoder.layer.2.attention.self\n",
      "roberta.encoder.layer.2.attention.self.query\n",
      "roberta.encoder.layer.2.attention.self.query.loras\n",
      "roberta.encoder.layer.2.attention.self.query.loras.rotten_tomatoes\n",
      "roberta.encoder.layer.2.attention.self.query.loras.rotten_tomatoes.lora_dropout\n",
      "roberta.encoder.layer.2.attention.self.key\n",
      "roberta.encoder.layer.2.attention.self.key.loras\n",
      "roberta.encoder.layer.2.attention.self.value\n",
      "roberta.encoder.layer.2.attention.self.value.loras\n",
      "roberta.encoder.layer.2.attention.self.value.loras.rotten_tomatoes\n",
      "roberta.encoder.layer.2.attention.self.value.loras.rotten_tomatoes.lora_dropout\n",
      "roberta.encoder.layer.2.attention.self.dropout\n",
      "roberta.encoder.layer.2.attention.self.prefix_tuning\n",
      "roberta.encoder.layer.2.attention.self.prefix_tuning.prefix_gates\n",
      "roberta.encoder.layer.2.attention.output\n",
      "roberta.encoder.layer.2.attention.output.dense\n",
      "roberta.encoder.layer.2.attention.output.LayerNorm\n",
      "roberta.encoder.layer.2.attention.output.dropout\n",
      "roberta.encoder.layer.2.attention.output.adapters\n",
      "roberta.encoder.layer.2.attention.output.adapter_fusion_layer\n",
      "roberta.encoder.layer.2.intermediate\n",
      "roberta.encoder.layer.2.intermediate.dense\n",
      "roberta.encoder.layer.2.intermediate.dense.loras\n",
      "roberta.encoder.layer.2.intermediate.intermediate_act_fn\n",
      "roberta.encoder.layer.2.output\n",
      "roberta.encoder.layer.2.output.dense\n",
      "roberta.encoder.layer.2.output.dense.loras\n",
      "roberta.encoder.layer.2.output.LayerNorm\n",
      "roberta.encoder.layer.2.output.dropout\n",
      "roberta.encoder.layer.2.output.adapters\n",
      "roberta.encoder.layer.2.output.adapters.rotten_tomatoes\n",
      "roberta.encoder.layer.2.output.adapters.rotten_tomatoes.non_linearity\n",
      "roberta.encoder.layer.2.output.adapters.rotten_tomatoes.non_linearity.f\n",
      "roberta.encoder.layer.2.output.adapters.rotten_tomatoes.adapter_down\n",
      "roberta.encoder.layer.2.output.adapters.rotten_tomatoes.adapter_down.0\n",
      "roberta.encoder.layer.2.output.adapters.rotten_tomatoes.adapter_up\n",
      "roberta.encoder.layer.2.output.adapter_fusion_layer\n",
      "roberta.encoder.layer.3\n",
      "roberta.encoder.layer.3.attention\n",
      "roberta.encoder.layer.3.attention.self\n",
      "roberta.encoder.layer.3.attention.self.query\n",
      "roberta.encoder.layer.3.attention.self.query.loras\n",
      "roberta.encoder.layer.3.attention.self.query.loras.rotten_tomatoes\n",
      "roberta.encoder.layer.3.attention.self.query.loras.rotten_tomatoes.lora_dropout\n",
      "roberta.encoder.layer.3.attention.self.key\n",
      "roberta.encoder.layer.3.attention.self.key.loras\n",
      "roberta.encoder.layer.3.attention.self.value\n",
      "roberta.encoder.layer.3.attention.self.value.loras\n",
      "roberta.encoder.layer.3.attention.self.value.loras.rotten_tomatoes\n",
      "roberta.encoder.layer.3.attention.self.value.loras.rotten_tomatoes.lora_dropout\n",
      "roberta.encoder.layer.3.attention.self.dropout\n",
      "roberta.encoder.layer.3.attention.self.prefix_tuning\n",
      "roberta.encoder.layer.3.attention.self.prefix_tuning.prefix_gates\n",
      "roberta.encoder.layer.3.attention.output\n",
      "roberta.encoder.layer.3.attention.output.dense\n",
      "roberta.encoder.layer.3.attention.output.LayerNorm\n",
      "roberta.encoder.layer.3.attention.output.dropout\n",
      "roberta.encoder.layer.3.attention.output.adapters\n",
      "roberta.encoder.layer.3.attention.output.adapter_fusion_layer\n",
      "roberta.encoder.layer.3.intermediate\n",
      "roberta.encoder.layer.3.intermediate.dense\n",
      "roberta.encoder.layer.3.intermediate.dense.loras\n",
      "roberta.encoder.layer.3.intermediate.intermediate_act_fn\n",
      "roberta.encoder.layer.3.output\n",
      "roberta.encoder.layer.3.output.dense\n",
      "roberta.encoder.layer.3.output.dense.loras\n",
      "roberta.encoder.layer.3.output.LayerNorm\n",
      "roberta.encoder.layer.3.output.dropout\n",
      "roberta.encoder.layer.3.output.adapters\n",
      "roberta.encoder.layer.3.output.adapters.rotten_tomatoes\n",
      "roberta.encoder.layer.3.output.adapters.rotten_tomatoes.non_linearity\n",
      "roberta.encoder.layer.3.output.adapters.rotten_tomatoes.non_linearity.f\n",
      "roberta.encoder.layer.3.output.adapters.rotten_tomatoes.adapter_down\n",
      "roberta.encoder.layer.3.output.adapters.rotten_tomatoes.adapter_down.0\n",
      "roberta.encoder.layer.3.output.adapters.rotten_tomatoes.adapter_up\n",
      "roberta.encoder.layer.3.output.adapter_fusion_layer\n",
      "roberta.encoder.layer.4\n",
      "roberta.encoder.layer.4.attention\n",
      "roberta.encoder.layer.4.attention.self\n",
      "roberta.encoder.layer.4.attention.self.query\n",
      "roberta.encoder.layer.4.attention.self.query.loras\n",
      "roberta.encoder.layer.4.attention.self.query.loras.rotten_tomatoes\n",
      "roberta.encoder.layer.4.attention.self.query.loras.rotten_tomatoes.lora_dropout\n",
      "roberta.encoder.layer.4.attention.self.key\n",
      "roberta.encoder.layer.4.attention.self.key.loras\n",
      "roberta.encoder.layer.4.attention.self.value\n",
      "roberta.encoder.layer.4.attention.self.value.loras\n",
      "roberta.encoder.layer.4.attention.self.value.loras.rotten_tomatoes\n",
      "roberta.encoder.layer.4.attention.self.value.loras.rotten_tomatoes.lora_dropout\n",
      "roberta.encoder.layer.4.attention.self.dropout\n",
      "roberta.encoder.layer.4.attention.self.prefix_tuning\n",
      "roberta.encoder.layer.4.attention.self.prefix_tuning.prefix_gates\n",
      "roberta.encoder.layer.4.attention.output\n",
      "roberta.encoder.layer.4.attention.output.dense\n",
      "roberta.encoder.layer.4.attention.output.LayerNorm\n",
      "roberta.encoder.layer.4.attention.output.dropout\n",
      "roberta.encoder.layer.4.attention.output.adapters\n",
      "roberta.encoder.layer.4.attention.output.adapter_fusion_layer\n",
      "roberta.encoder.layer.4.intermediate\n",
      "roberta.encoder.layer.4.intermediate.dense\n",
      "roberta.encoder.layer.4.intermediate.dense.loras\n",
      "roberta.encoder.layer.4.intermediate.intermediate_act_fn\n",
      "roberta.encoder.layer.4.output\n",
      "roberta.encoder.layer.4.output.dense\n",
      "roberta.encoder.layer.4.output.dense.loras\n",
      "roberta.encoder.layer.4.output.LayerNorm\n",
      "roberta.encoder.layer.4.output.dropout\n",
      "roberta.encoder.layer.4.output.adapters\n",
      "roberta.encoder.layer.4.output.adapters.rotten_tomatoes\n",
      "roberta.encoder.layer.4.output.adapters.rotten_tomatoes.non_linearity\n",
      "roberta.encoder.layer.4.output.adapters.rotten_tomatoes.non_linearity.f\n",
      "roberta.encoder.layer.4.output.adapters.rotten_tomatoes.adapter_down\n",
      "roberta.encoder.layer.4.output.adapters.rotten_tomatoes.adapter_down.0\n",
      "roberta.encoder.layer.4.output.adapters.rotten_tomatoes.adapter_up\n",
      "roberta.encoder.layer.4.output.adapter_fusion_layer\n",
      "roberta.encoder.layer.5\n",
      "roberta.encoder.layer.5.attention\n",
      "roberta.encoder.layer.5.attention.self\n",
      "roberta.encoder.layer.5.attention.self.query\n",
      "roberta.encoder.layer.5.attention.self.query.loras\n",
      "roberta.encoder.layer.5.attention.self.query.loras.rotten_tomatoes\n",
      "roberta.encoder.layer.5.attention.self.query.loras.rotten_tomatoes.lora_dropout\n",
      "roberta.encoder.layer.5.attention.self.key\n",
      "roberta.encoder.layer.5.attention.self.key.loras\n",
      "roberta.encoder.layer.5.attention.self.value\n",
      "roberta.encoder.layer.5.attention.self.value.loras\n",
      "roberta.encoder.layer.5.attention.self.value.loras.rotten_tomatoes\n",
      "roberta.encoder.layer.5.attention.self.value.loras.rotten_tomatoes.lora_dropout\n",
      "roberta.encoder.layer.5.attention.self.dropout\n",
      "roberta.encoder.layer.5.attention.self.prefix_tuning\n",
      "roberta.encoder.layer.5.attention.self.prefix_tuning.prefix_gates\n",
      "roberta.encoder.layer.5.attention.output\n",
      "roberta.encoder.layer.5.attention.output.dense\n",
      "roberta.encoder.layer.5.attention.output.LayerNorm\n",
      "roberta.encoder.layer.5.attention.output.dropout\n",
      "roberta.encoder.layer.5.attention.output.adapters\n",
      "roberta.encoder.layer.5.attention.output.adapter_fusion_layer\n",
      "roberta.encoder.layer.5.intermediate\n",
      "roberta.encoder.layer.5.intermediate.dense\n",
      "roberta.encoder.layer.5.intermediate.dense.loras\n",
      "roberta.encoder.layer.5.intermediate.intermediate_act_fn\n",
      "roberta.encoder.layer.5.output\n",
      "roberta.encoder.layer.5.output.dense\n",
      "roberta.encoder.layer.5.output.dense.loras\n",
      "roberta.encoder.layer.5.output.LayerNorm\n",
      "roberta.encoder.layer.5.output.dropout\n",
      "roberta.encoder.layer.5.output.adapters\n",
      "roberta.encoder.layer.5.output.adapters.rotten_tomatoes\n",
      "roberta.encoder.layer.5.output.adapters.rotten_tomatoes.non_linearity\n",
      "roberta.encoder.layer.5.output.adapters.rotten_tomatoes.non_linearity.f\n",
      "roberta.encoder.layer.5.output.adapters.rotten_tomatoes.adapter_down\n",
      "roberta.encoder.layer.5.output.adapters.rotten_tomatoes.adapter_down.0\n",
      "roberta.encoder.layer.5.output.adapters.rotten_tomatoes.adapter_up\n",
      "roberta.encoder.layer.5.output.adapter_fusion_layer\n",
      "roberta.encoder.layer.6\n",
      "roberta.encoder.layer.6.attention\n",
      "roberta.encoder.layer.6.attention.self\n",
      "roberta.encoder.layer.6.attention.self.query\n",
      "roberta.encoder.layer.6.attention.self.query.loras\n",
      "roberta.encoder.layer.6.attention.self.key\n",
      "roberta.encoder.layer.6.attention.self.key.loras\n",
      "roberta.encoder.layer.6.attention.self.value\n",
      "roberta.encoder.layer.6.attention.self.value.loras\n",
      "roberta.encoder.layer.6.attention.self.dropout\n",
      "roberta.encoder.layer.6.attention.self.prefix_tuning\n",
      "roberta.encoder.layer.6.attention.self.prefix_tuning.prefix_gates\n",
      "roberta.encoder.layer.6.attention.output\n",
      "roberta.encoder.layer.6.attention.output.dense\n",
      "roberta.encoder.layer.6.attention.output.LayerNorm\n",
      "roberta.encoder.layer.6.attention.output.dropout\n",
      "roberta.encoder.layer.6.attention.output.adapters\n",
      "roberta.encoder.layer.6.attention.output.adapter_fusion_layer\n",
      "roberta.encoder.layer.6.intermediate\n",
      "roberta.encoder.layer.6.intermediate.dense\n",
      "roberta.encoder.layer.6.intermediate.dense.loras\n",
      "roberta.encoder.layer.6.intermediate.intermediate_act_fn\n",
      "roberta.encoder.layer.6.output\n",
      "roberta.encoder.layer.6.output.dense\n",
      "roberta.encoder.layer.6.output.dense.loras\n",
      "roberta.encoder.layer.6.output.LayerNorm\n",
      "roberta.encoder.layer.6.output.dropout\n",
      "roberta.encoder.layer.6.output.adapters\n",
      "roberta.encoder.layer.6.output.adapter_fusion_layer\n",
      "roberta.encoder.layer.7\n",
      "roberta.encoder.layer.7.attention\n",
      "roberta.encoder.layer.7.attention.self\n",
      "roberta.encoder.layer.7.attention.self.query\n",
      "roberta.encoder.layer.7.attention.self.query.loras\n",
      "roberta.encoder.layer.7.attention.self.key\n",
      "roberta.encoder.layer.7.attention.self.key.loras\n",
      "roberta.encoder.layer.7.attention.self.value\n",
      "roberta.encoder.layer.7.attention.self.value.loras\n",
      "roberta.encoder.layer.7.attention.self.dropout\n",
      "roberta.encoder.layer.7.attention.self.prefix_tuning\n",
      "roberta.encoder.layer.7.attention.self.prefix_tuning.prefix_gates\n",
      "roberta.encoder.layer.7.attention.output\n",
      "roberta.encoder.layer.7.attention.output.dense\n",
      "roberta.encoder.layer.7.attention.output.LayerNorm\n",
      "roberta.encoder.layer.7.attention.output.dropout\n",
      "roberta.encoder.layer.7.attention.output.adapters\n",
      "roberta.encoder.layer.7.attention.output.adapter_fusion_layer\n",
      "roberta.encoder.layer.7.intermediate\n",
      "roberta.encoder.layer.7.intermediate.dense\n",
      "roberta.encoder.layer.7.intermediate.dense.loras\n",
      "roberta.encoder.layer.7.intermediate.intermediate_act_fn\n",
      "roberta.encoder.layer.7.output\n",
      "roberta.encoder.layer.7.output.dense\n",
      "roberta.encoder.layer.7.output.dense.loras\n",
      "roberta.encoder.layer.7.output.LayerNorm\n",
      "roberta.encoder.layer.7.output.dropout\n",
      "roberta.encoder.layer.7.output.adapters\n",
      "roberta.encoder.layer.7.output.adapter_fusion_layer\n",
      "roberta.encoder.layer.8\n",
      "roberta.encoder.layer.8.attention\n",
      "roberta.encoder.layer.8.attention.self\n",
      "roberta.encoder.layer.8.attention.self.query\n",
      "roberta.encoder.layer.8.attention.self.query.loras\n",
      "roberta.encoder.layer.8.attention.self.key\n",
      "roberta.encoder.layer.8.attention.self.key.loras\n",
      "roberta.encoder.layer.8.attention.self.value\n",
      "roberta.encoder.layer.8.attention.self.value.loras\n",
      "roberta.encoder.layer.8.attention.self.dropout\n",
      "roberta.encoder.layer.8.attention.self.prefix_tuning\n",
      "roberta.encoder.layer.8.attention.self.prefix_tuning.prefix_gates\n",
      "roberta.encoder.layer.8.attention.output\n",
      "roberta.encoder.layer.8.attention.output.dense\n",
      "roberta.encoder.layer.8.attention.output.LayerNorm\n",
      "roberta.encoder.layer.8.attention.output.dropout\n",
      "roberta.encoder.layer.8.attention.output.adapters\n",
      "roberta.encoder.layer.8.attention.output.adapter_fusion_layer\n",
      "roberta.encoder.layer.8.intermediate\n",
      "roberta.encoder.layer.8.intermediate.dense\n",
      "roberta.encoder.layer.8.intermediate.dense.loras\n",
      "roberta.encoder.layer.8.intermediate.intermediate_act_fn\n",
      "roberta.encoder.layer.8.output\n",
      "roberta.encoder.layer.8.output.dense\n",
      "roberta.encoder.layer.8.output.dense.loras\n",
      "roberta.encoder.layer.8.output.LayerNorm\n",
      "roberta.encoder.layer.8.output.dropout\n",
      "roberta.encoder.layer.8.output.adapters\n",
      "roberta.encoder.layer.8.output.adapter_fusion_layer\n",
      "roberta.encoder.layer.9\n",
      "roberta.encoder.layer.9.attention\n",
      "roberta.encoder.layer.9.attention.self\n",
      "roberta.encoder.layer.9.attention.self.query\n",
      "roberta.encoder.layer.9.attention.self.query.loras\n",
      "roberta.encoder.layer.9.attention.self.key\n",
      "roberta.encoder.layer.9.attention.self.key.loras\n",
      "roberta.encoder.layer.9.attention.self.value\n",
      "roberta.encoder.layer.9.attention.self.value.loras\n",
      "roberta.encoder.layer.9.attention.self.dropout\n",
      "roberta.encoder.layer.9.attention.self.prefix_tuning\n",
      "roberta.encoder.layer.9.attention.self.prefix_tuning.prefix_gates\n",
      "roberta.encoder.layer.9.attention.output\n",
      "roberta.encoder.layer.9.attention.output.dense\n",
      "roberta.encoder.layer.9.attention.output.LayerNorm\n",
      "roberta.encoder.layer.9.attention.output.dropout\n",
      "roberta.encoder.layer.9.attention.output.adapters\n",
      "roberta.encoder.layer.9.attention.output.adapter_fusion_layer\n",
      "roberta.encoder.layer.9.intermediate\n",
      "roberta.encoder.layer.9.intermediate.dense\n",
      "roberta.encoder.layer.9.intermediate.dense.loras\n",
      "roberta.encoder.layer.9.intermediate.intermediate_act_fn\n",
      "roberta.encoder.layer.9.output\n",
      "roberta.encoder.layer.9.output.dense\n",
      "roberta.encoder.layer.9.output.dense.loras\n",
      "roberta.encoder.layer.9.output.LayerNorm\n",
      "roberta.encoder.layer.9.output.dropout\n",
      "roberta.encoder.layer.9.output.adapters\n",
      "roberta.encoder.layer.9.output.adapter_fusion_layer\n",
      "roberta.encoder.layer.10\n",
      "roberta.encoder.layer.10.attention\n",
      "roberta.encoder.layer.10.attention.self\n",
      "roberta.encoder.layer.10.attention.self.query\n",
      "roberta.encoder.layer.10.attention.self.query.loras\n",
      "roberta.encoder.layer.10.attention.self.key\n",
      "roberta.encoder.layer.10.attention.self.key.loras\n",
      "roberta.encoder.layer.10.attention.self.value\n",
      "roberta.encoder.layer.10.attention.self.value.loras\n",
      "roberta.encoder.layer.10.attention.self.dropout\n",
      "roberta.encoder.layer.10.attention.self.prefix_tuning\n",
      "roberta.encoder.layer.10.attention.self.prefix_tuning.prefix_gates\n",
      "roberta.encoder.layer.10.attention.output\n",
      "roberta.encoder.layer.10.attention.output.dense\n",
      "roberta.encoder.layer.10.attention.output.LayerNorm\n",
      "roberta.encoder.layer.10.attention.output.dropout\n",
      "roberta.encoder.layer.10.attention.output.adapters\n",
      "roberta.encoder.layer.10.attention.output.adapter_fusion_layer\n",
      "roberta.encoder.layer.10.intermediate\n",
      "roberta.encoder.layer.10.intermediate.dense\n",
      "roberta.encoder.layer.10.intermediate.dense.loras\n",
      "roberta.encoder.layer.10.intermediate.intermediate_act_fn\n",
      "roberta.encoder.layer.10.output\n",
      "roberta.encoder.layer.10.output.dense\n",
      "roberta.encoder.layer.10.output.dense.loras\n",
      "roberta.encoder.layer.10.output.LayerNorm\n",
      "roberta.encoder.layer.10.output.dropout\n",
      "roberta.encoder.layer.10.output.adapters\n",
      "roberta.encoder.layer.10.output.adapter_fusion_layer\n",
      "roberta.encoder.layer.11\n",
      "roberta.encoder.layer.11.attention\n",
      "roberta.encoder.layer.11.attention.self\n",
      "roberta.encoder.layer.11.attention.self.query\n",
      "roberta.encoder.layer.11.attention.self.query.loras\n",
      "roberta.encoder.layer.11.attention.self.key\n",
      "roberta.encoder.layer.11.attention.self.key.loras\n",
      "roberta.encoder.layer.11.attention.self.value\n",
      "roberta.encoder.layer.11.attention.self.value.loras\n",
      "roberta.encoder.layer.11.attention.self.dropout\n",
      "roberta.encoder.layer.11.attention.self.prefix_tuning\n",
      "roberta.encoder.layer.11.attention.self.prefix_tuning.prefix_gates\n",
      "roberta.encoder.layer.11.attention.output\n",
      "roberta.encoder.layer.11.attention.output.dense\n",
      "roberta.encoder.layer.11.attention.output.LayerNorm\n",
      "roberta.encoder.layer.11.attention.output.dropout\n",
      "roberta.encoder.layer.11.attention.output.adapters\n",
      "roberta.encoder.layer.11.attention.output.adapter_fusion_layer\n",
      "roberta.encoder.layer.11.intermediate\n",
      "roberta.encoder.layer.11.intermediate.dense\n",
      "roberta.encoder.layer.11.intermediate.dense.loras\n",
      "roberta.encoder.layer.11.intermediate.intermediate_act_fn\n",
      "roberta.encoder.layer.11.output\n",
      "roberta.encoder.layer.11.output.dense\n",
      "roberta.encoder.layer.11.output.dense.loras\n",
      "roberta.encoder.layer.11.output.LayerNorm\n",
      "roberta.encoder.layer.11.output.dropout\n",
      "roberta.encoder.layer.11.output.adapters\n",
      "roberta.encoder.layer.11.output.adapter_fusion_layer\n",
      "roberta.pooler\n",
      "roberta.pooler.dense\n",
      "roberta.pooler.activation\n",
      "roberta.invertible_adapters\n",
      "roberta.shared_parameters\n",
      "roberta.prompt_tuning\n",
      "roberta.prompt_tuning.prompt_tunings\n",
      "heads\n",
      "heads.default\n",
      "heads.default.0\n",
      "heads.default.1\n",
      "heads.default.1.f\n",
      "heads.default.2\n",
      "heads.default.3\n"
     ]
    }
   ],
   "source": [
    "# 打印模型中模块的名字\n",
    "for name, module in model.named_modules():\n",
    "    print(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_trainable_parameters(model):\n",
    "    trainable_params = 0\n",
    "    all_param = 0\n",
    "    for _, param in model.named_parameters():\n",
    "        all_param += param.numel()\n",
    "        if param.requires_grad:\n",
    "            trainable_params += param.numel()\n",
    "    print(\n",
    "        f\"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param:.2f}\"\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=========================================================================================================\n",
      "Layer (type:depth-idx)                                                           Param #\n",
      "=========================================================================================================\n",
      "RobertaModel                                                                     --\n",
      "├─RobertaEmbeddings: 1-1                                                         --\n",
      "│    └─Embedding: 2-1                                                            38,603,520\n",
      "│    └─Embedding: 2-2                                                            394,752\n",
      "│    └─Embedding: 2-3                                                            768\n",
      "│    └─LayerNorm: 2-4                                                            1,536\n",
      "│    └─Dropout: 2-5                                                              --\n",
      "├─RobertaEncoder: 1-2                                                            --\n",
      "│    └─ModuleList: 2-6                                                           --\n",
      "│    │    └─RobertaLayer: 3-1                                                    --\n",
      "│    │    │    └─RobertaAttention: 4-1                                           --\n",
      "│    │    │    │    └─RobertaSelfAttentionWithAdapters: 5-1                      --\n",
      "│    │    │    │    │    └─LoRALinear: 6-1                                       590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-1                                  --\n",
      "│    │    │    │    │    └─LoRALinear: 6-2                                       590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-2                                  --\n",
      "│    │    │    │    │    └─LoRALinear: 6-3                                       590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-3                                  --\n",
      "│    │    │    │    │    └─Dropout: 6-4                                          --\n",
      "│    │    │    │    │    └─PrefixTuningLayer: 6-5                                --\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-4                                  --\n",
      "│    │    │    │    │    │    └─PrefixTuningPool: 7-5                            --\n",
      "│    │    │    │    │    │    │    └─ModuleDict: 8-1                             --\n",
      "│    │    │    │    └─RobertaSelfOutputWithAdapters: 5-2                         --\n",
      "│    │    │    │    │    └─Linear: 6-6                                           590,592\n",
      "│    │    │    │    │    └─LayerNorm: 6-7                                        1,536\n",
      "│    │    │    │    │    └─Dropout: 6-8                                          --\n",
      "│    │    │    │    │    └─ModuleDict: 6-9                                       --\n",
      "│    │    │    │    │    │    └─Adapter: 7-6                                     --\n",
      "│    │    │    │    │    │    │    └─Activation_Function_Class: 8-2              --\n",
      "│    │    │    │    │    │    │    │    └─GELUActivation: 9-1                    --\n",
      "│    │    │    │    │    │    │    └─Sequential: 8-3                             --\n",
      "│    │    │    │    │    │    │    │    └─Linear: 9-2                            36,912\n",
      "│    │    │    │    │    │    │    │    └─Activation_Function_Class: 9-3         --\n",
      "│    │    │    │    │    │    │    │    │    └─GELUActivation: 10-1              --\n",
      "│    │    │    │    │    │    │    └─Linear: 8-4                                 37,632\n",
      "│    │    │    │    │    └─ModuleDict: 6-10                                      --\n",
      "│    │    │    └─RobertaIntermediate: 4-2                                        --\n",
      "│    │    │    │    └─LoRALinear: 5-3                                            2,362,368\n",
      "│    │    │    │    │    └─ModuleDict: 6-11                                      --\n",
      "│    │    │    │    └─GELUActivation: 5-4                                        --\n",
      "│    │    │    └─RobertaOutputWithAdapters: 4-3                                  --\n",
      "│    │    │    │    └─LoRALinear: 5-5                                            2,360,064\n",
      "│    │    │    │    │    └─ModuleDict: 6-12                                      --\n",
      "│    │    │    │    └─LayerNorm: 5-6                                             1,536\n",
      "│    │    │    │    └─Dropout: 5-7                                               --\n",
      "│    │    │    │    └─ModuleDict: 5-8                                            --\n",
      "│    │    │    │    │    └─Adapter: 6-13                                         --\n",
      "│    │    │    │    │    │    └─Activation_Function_Class: 7-7                   --\n",
      "│    │    │    │    │    │    │    └─GELUActivation: 8-5                         --\n",
      "│    │    │    │    │    │    └─Sequential: 7-8                                  --\n",
      "│    │    │    │    │    │    │    └─Linear: 8-6                                 36,912\n",
      "│    │    │    │    │    │    │    └─Activation_Function_Class: 8-7              --\n",
      "│    │    │    │    │    │    │    │    └─GELUActivation: 9-4                    --\n",
      "│    │    │    │    │    │    └─Linear: 7-9                                      37,632\n",
      "│    │    │    │    └─ModuleDict: 5-9                                            --\n",
      "│    │    └─RobertaLayer: 3-2                                                    --\n",
      "│    │    │    └─RobertaAttention: 4-4                                           --\n",
      "│    │    │    │    └─RobertaSelfAttentionWithAdapters: 5-10                     --\n",
      "│    │    │    │    │    └─LoRALinear: 6-14                                      590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-10                                 --\n",
      "│    │    │    │    │    └─LoRALinear: 6-15                                      590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-11                                 --\n",
      "│    │    │    │    │    └─LoRALinear: 6-16                                      590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-12                                 --\n",
      "│    │    │    │    │    └─Dropout: 6-17                                         --\n",
      "│    │    │    │    │    └─PrefixTuningLayer: 6-18                               --\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-13                                 --\n",
      "│    │    │    │    │    │    └─PrefixTuningPool: 7-14                           --\n",
      "│    │    │    │    │    │    │    └─ModuleDict: 8-8                             --\n",
      "│    │    │    │    └─RobertaSelfOutputWithAdapters: 5-11                        --\n",
      "│    │    │    │    │    └─Linear: 6-19                                          590,592\n",
      "│    │    │    │    │    └─LayerNorm: 6-20                                       1,536\n",
      "│    │    │    │    │    └─Dropout: 6-21                                         --\n",
      "│    │    │    │    │    └─ModuleDict: 6-22                                      --\n",
      "│    │    │    │    │    │    └─Adapter: 7-15                                    --\n",
      "│    │    │    │    │    │    │    └─Activation_Function_Class: 8-9              --\n",
      "│    │    │    │    │    │    │    │    └─GELUActivation: 9-5                    --\n",
      "│    │    │    │    │    │    │    └─Sequential: 8-10                            --\n",
      "│    │    │    │    │    │    │    │    └─Linear: 9-6                            36,912\n",
      "│    │    │    │    │    │    │    │    └─Activation_Function_Class: 9-7         --\n",
      "│    │    │    │    │    │    │    │    │    └─GELUActivation: 10-2              --\n",
      "│    │    │    │    │    │    │    └─Linear: 8-11                                37,632\n",
      "│    │    │    │    │    └─ModuleDict: 6-23                                      --\n",
      "│    │    │    └─RobertaIntermediate: 4-5                                        --\n",
      "│    │    │    │    └─LoRALinear: 5-12                                           2,362,368\n",
      "│    │    │    │    │    └─ModuleDict: 6-24                                      --\n",
      "│    │    │    │    └─GELUActivation: 5-13                                       --\n",
      "│    │    │    └─RobertaOutputWithAdapters: 4-6                                  --\n",
      "│    │    │    │    └─LoRALinear: 5-14                                           2,360,064\n",
      "│    │    │    │    │    └─ModuleDict: 6-25                                      --\n",
      "│    │    │    │    └─LayerNorm: 5-15                                            1,536\n",
      "│    │    │    │    └─Dropout: 5-16                                              --\n",
      "│    │    │    │    └─ModuleDict: 5-17                                           --\n",
      "│    │    │    │    │    └─Adapter: 6-26                                         --\n",
      "│    │    │    │    │    │    └─Activation_Function_Class: 7-16                  --\n",
      "│    │    │    │    │    │    │    └─GELUActivation: 8-12                        --\n",
      "│    │    │    │    │    │    └─Sequential: 7-17                                 --\n",
      "│    │    │    │    │    │    │    └─Linear: 8-13                                36,912\n",
      "│    │    │    │    │    │    │    └─Activation_Function_Class: 8-14             --\n",
      "│    │    │    │    │    │    │    │    └─GELUActivation: 9-8                    --\n",
      "│    │    │    │    │    │    └─Linear: 7-18                                     37,632\n",
      "│    │    │    │    └─ModuleDict: 5-18                                           --\n",
      "│    │    └─RobertaLayer: 3-3                                                    --\n",
      "│    │    │    └─RobertaAttention: 4-7                                           --\n",
      "│    │    │    │    └─RobertaSelfAttentionWithAdapters: 5-19                     --\n",
      "│    │    │    │    │    └─LoRALinear: 6-27                                      590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-19                                 --\n",
      "│    │    │    │    │    └─LoRALinear: 6-28                                      590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-20                                 --\n",
      "│    │    │    │    │    └─LoRALinear: 6-29                                      590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-21                                 --\n",
      "│    │    │    │    │    └─Dropout: 6-30                                         --\n",
      "│    │    │    │    │    └─PrefixTuningLayer: 6-31                               --\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-22                                 --\n",
      "│    │    │    │    │    │    └─PrefixTuningPool: 7-23                           --\n",
      "│    │    │    │    │    │    │    └─ModuleDict: 8-15                            --\n",
      "│    │    │    │    └─RobertaSelfOutputWithAdapters: 5-20                        --\n",
      "│    │    │    │    │    └─Linear: 6-32                                          590,592\n",
      "│    │    │    │    │    └─LayerNorm: 6-33                                       1,536\n",
      "│    │    │    │    │    └─Dropout: 6-34                                         --\n",
      "│    │    │    │    │    └─ModuleDict: 6-35                                      --\n",
      "│    │    │    │    │    │    └─Adapter: 7-24                                    --\n",
      "│    │    │    │    │    │    │    └─Activation_Function_Class: 8-16             --\n",
      "│    │    │    │    │    │    │    │    └─GELUActivation: 9-9                    --\n",
      "│    │    │    │    │    │    │    └─Sequential: 8-17                            --\n",
      "│    │    │    │    │    │    │    │    └─Linear: 9-10                           36,912\n",
      "│    │    │    │    │    │    │    │    └─Activation_Function_Class: 9-11        --\n",
      "│    │    │    │    │    │    │    │    │    └─GELUActivation: 10-3              --\n",
      "│    │    │    │    │    │    │    └─Linear: 8-18                                37,632\n",
      "│    │    │    │    │    └─ModuleDict: 6-36                                      --\n",
      "│    │    │    └─RobertaIntermediate: 4-8                                        --\n",
      "│    │    │    │    └─LoRALinear: 5-21                                           2,362,368\n",
      "│    │    │    │    │    └─ModuleDict: 6-37                                      --\n",
      "│    │    │    │    └─GELUActivation: 5-22                                       --\n",
      "│    │    │    └─RobertaOutputWithAdapters: 4-9                                  --\n",
      "│    │    │    │    └─LoRALinear: 5-23                                           2,360,064\n",
      "│    │    │    │    │    └─ModuleDict: 6-38                                      --\n",
      "│    │    │    │    └─LayerNorm: 5-24                                            1,536\n",
      "│    │    │    │    └─Dropout: 5-25                                              --\n",
      "│    │    │    │    └─ModuleDict: 5-26                                           --\n",
      "│    │    │    │    │    └─Adapter: 6-39                                         --\n",
      "│    │    │    │    │    │    └─Activation_Function_Class: 7-25                  --\n",
      "│    │    │    │    │    │    │    └─GELUActivation: 8-19                        --\n",
      "│    │    │    │    │    │    └─Sequential: 7-26                                 --\n",
      "│    │    │    │    │    │    │    └─Linear: 8-20                                36,912\n",
      "│    │    │    │    │    │    │    └─Activation_Function_Class: 8-21             --\n",
      "│    │    │    │    │    │    │    │    └─GELUActivation: 9-12                   --\n",
      "│    │    │    │    │    │    └─Linear: 7-27                                     37,632\n",
      "│    │    │    │    └─ModuleDict: 5-27                                           --\n",
      "│    │    └─RobertaLayer: 3-4                                                    --\n",
      "│    │    │    └─RobertaAttention: 4-10                                          --\n",
      "│    │    │    │    └─RobertaSelfAttentionWithAdapters: 5-28                     --\n",
      "│    │    │    │    │    └─LoRALinear: 6-40                                      590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-28                                 --\n",
      "│    │    │    │    │    └─LoRALinear: 6-41                                      590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-29                                 --\n",
      "│    │    │    │    │    └─LoRALinear: 6-42                                      590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-30                                 --\n",
      "│    │    │    │    │    └─Dropout: 6-43                                         --\n",
      "│    │    │    │    │    └─PrefixTuningLayer: 6-44                               --\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-31                                 --\n",
      "│    │    │    │    │    │    └─PrefixTuningPool: 7-32                           --\n",
      "│    │    │    │    │    │    │    └─ModuleDict: 8-22                            --\n",
      "│    │    │    │    └─RobertaSelfOutputWithAdapters: 5-29                        --\n",
      "│    │    │    │    │    └─Linear: 6-45                                          590,592\n",
      "│    │    │    │    │    └─LayerNorm: 6-46                                       1,536\n",
      "│    │    │    │    │    └─Dropout: 6-47                                         --\n",
      "│    │    │    │    │    └─ModuleDict: 6-48                                      --\n",
      "│    │    │    │    │    │    └─Adapter: 7-33                                    --\n",
      "│    │    │    │    │    │    │    └─Activation_Function_Class: 8-23             --\n",
      "│    │    │    │    │    │    │    │    └─GELUActivation: 9-13                   --\n",
      "│    │    │    │    │    │    │    └─Sequential: 8-24                            --\n",
      "│    │    │    │    │    │    │    │    └─Linear: 9-14                           36,912\n",
      "│    │    │    │    │    │    │    │    └─Activation_Function_Class: 9-15        --\n",
      "│    │    │    │    │    │    │    │    │    └─GELUActivation: 10-4              --\n",
      "│    │    │    │    │    │    │    └─Linear: 8-25                                37,632\n",
      "│    │    │    │    │    └─ModuleDict: 6-49                                      --\n",
      "│    │    │    └─RobertaIntermediate: 4-11                                       --\n",
      "│    │    │    │    └─LoRALinear: 5-30                                           2,362,368\n",
      "│    │    │    │    │    └─ModuleDict: 6-50                                      --\n",
      "│    │    │    │    └─GELUActivation: 5-31                                       --\n",
      "│    │    │    └─RobertaOutputWithAdapters: 4-12                                 --\n",
      "│    │    │    │    └─LoRALinear: 5-32                                           2,360,064\n",
      "│    │    │    │    │    └─ModuleDict: 6-51                                      --\n",
      "│    │    │    │    └─LayerNorm: 5-33                                            1,536\n",
      "│    │    │    │    └─Dropout: 5-34                                              --\n",
      "│    │    │    │    └─ModuleDict: 5-35                                           --\n",
      "│    │    │    │    │    └─Adapter: 6-52                                         --\n",
      "│    │    │    │    │    │    └─Activation_Function_Class: 7-34                  --\n",
      "│    │    │    │    │    │    │    └─GELUActivation: 8-26                        --\n",
      "│    │    │    │    │    │    └─Sequential: 7-35                                 --\n",
      "│    │    │    │    │    │    │    └─Linear: 8-27                                36,912\n",
      "│    │    │    │    │    │    │    └─Activation_Function_Class: 8-28             --\n",
      "│    │    │    │    │    │    │    │    └─GELUActivation: 9-16                   --\n",
      "│    │    │    │    │    │    └─Linear: 7-36                                     37,632\n",
      "│    │    │    │    └─ModuleDict: 5-36                                           --\n",
      "│    │    └─RobertaLayer: 3-5                                                    --\n",
      "│    │    │    └─RobertaAttention: 4-13                                          --\n",
      "│    │    │    │    └─RobertaSelfAttentionWithAdapters: 5-37                     --\n",
      "│    │    │    │    │    └─LoRALinear: 6-53                                      590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-37                                 --\n",
      "│    │    │    │    │    │    │    └─LoRA: 8-29                                  12,288\n",
      "│    │    │    │    │    └─LoRALinear: 6-54                                      590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-38                                 --\n",
      "│    │    │    │    │    └─LoRALinear: 6-55                                      590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-39                                 --\n",
      "│    │    │    │    │    │    │    └─LoRA: 8-30                                  12,288\n",
      "│    │    │    │    │    └─Dropout: 6-56                                         --\n",
      "│    │    │    │    │    └─PrefixTuningLayer: 6-57                               --\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-40                                 --\n",
      "│    │    │    │    │    │    └─PrefixTuningPool: 7-41                           --\n",
      "│    │    │    │    │    │    │    └─ModuleDict: 8-31                            --\n",
      "│    │    │    │    └─RobertaSelfOutputWithAdapters: 5-38                        --\n",
      "│    │    │    │    │    └─Linear: 6-58                                          590,592\n",
      "│    │    │    │    │    └─LayerNorm: 6-59                                       1,536\n",
      "│    │    │    │    │    └─Dropout: 6-60                                         --\n",
      "│    │    │    │    │    └─ModuleDict: 6-61                                      --\n",
      "│    │    │    │    │    └─ModuleDict: 6-62                                      --\n",
      "│    │    │    └─RobertaIntermediate: 4-14                                       --\n",
      "│    │    │    │    └─LoRALinear: 5-39                                           2,362,368\n",
      "│    │    │    │    │    └─ModuleDict: 6-63                                      --\n",
      "│    │    │    │    └─GELUActivation: 5-40                                       --\n",
      "│    │    │    └─RobertaOutputWithAdapters: 4-15                                 --\n",
      "│    │    │    │    └─LoRALinear: 5-41                                           2,360,064\n",
      "│    │    │    │    │    └─ModuleDict: 6-64                                      --\n",
      "│    │    │    │    └─LayerNorm: 5-42                                            1,536\n",
      "│    │    │    │    └─Dropout: 5-43                                              --\n",
      "│    │    │    │    └─ModuleDict: 5-44                                           --\n",
      "│    │    │    │    └─ModuleDict: 5-45                                           --\n",
      "│    │    └─RobertaLayer: 3-6                                                    --\n",
      "│    │    │    └─RobertaAttention: 4-16                                          --\n",
      "│    │    │    │    └─RobertaSelfAttentionWithAdapters: 5-46                     --\n",
      "│    │    │    │    │    └─LoRALinear: 6-65                                      590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-42                                 --\n",
      "│    │    │    │    │    │    │    └─LoRA: 8-32                                  12,288\n",
      "│    │    │    │    │    └─LoRALinear: 6-66                                      590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-43                                 --\n",
      "│    │    │    │    │    └─LoRALinear: 6-67                                      590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-44                                 --\n",
      "│    │    │    │    │    │    │    └─LoRA: 8-33                                  12,288\n",
      "│    │    │    │    │    └─Dropout: 6-68                                         --\n",
      "│    │    │    │    │    └─PrefixTuningLayer: 6-69                               --\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-45                                 --\n",
      "│    │    │    │    │    │    └─PrefixTuningPool: 7-46                           --\n",
      "│    │    │    │    │    │    │    └─ModuleDict: 8-34                            --\n",
      "│    │    │    │    └─RobertaSelfOutputWithAdapters: 5-47                        --\n",
      "│    │    │    │    │    └─Linear: 6-70                                          590,592\n",
      "│    │    │    │    │    └─LayerNorm: 6-71                                       1,536\n",
      "│    │    │    │    │    └─Dropout: 6-72                                         --\n",
      "│    │    │    │    │    └─ModuleDict: 6-73                                      --\n",
      "│    │    │    │    │    └─ModuleDict: 6-74                                      --\n",
      "│    │    │    └─RobertaIntermediate: 4-17                                       --\n",
      "│    │    │    │    └─LoRALinear: 5-48                                           2,362,368\n",
      "│    │    │    │    │    └─ModuleDict: 6-75                                      --\n",
      "│    │    │    │    └─GELUActivation: 5-49                                       --\n",
      "│    │    │    └─RobertaOutputWithAdapters: 4-18                                 --\n",
      "│    │    │    │    └─LoRALinear: 5-50                                           2,360,064\n",
      "│    │    │    │    │    └─ModuleDict: 6-76                                      --\n",
      "│    │    │    │    └─LayerNorm: 5-51                                            1,536\n",
      "│    │    │    │    └─Dropout: 5-52                                              --\n",
      "│    │    │    │    └─ModuleDict: 5-53                                           --\n",
      "│    │    │    │    └─ModuleDict: 5-54                                           --\n",
      "│    │    └─RobertaLayer: 3-7                                                    --\n",
      "│    │    │    └─RobertaAttention: 4-19                                          --\n",
      "│    │    │    │    └─RobertaSelfAttentionWithAdapters: 5-55                     --\n",
      "│    │    │    │    │    └─LoRALinear: 6-77                                      590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-47                                 --\n",
      "│    │    │    │    │    │    │    └─LoRA: 8-35                                  12,288\n",
      "│    │    │    │    │    └─LoRALinear: 6-78                                      590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-48                                 --\n",
      "│    │    │    │    │    └─LoRALinear: 6-79                                      590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-49                                 --\n",
      "│    │    │    │    │    │    │    └─LoRA: 8-36                                  12,288\n",
      "│    │    │    │    │    └─Dropout: 6-80                                         --\n",
      "│    │    │    │    │    └─PrefixTuningLayer: 6-81                               --\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-50                                 --\n",
      "│    │    │    │    │    │    └─PrefixTuningPool: 7-51                           --\n",
      "│    │    │    │    │    │    │    └─ModuleDict: 8-37                            --\n",
      "│    │    │    │    └─RobertaSelfOutputWithAdapters: 5-56                        --\n",
      "│    │    │    │    │    └─Linear: 6-82                                          590,592\n",
      "│    │    │    │    │    └─LayerNorm: 6-83                                       1,536\n",
      "│    │    │    │    │    └─Dropout: 6-84                                         --\n",
      "│    │    │    │    │    └─ModuleDict: 6-85                                      --\n",
      "│    │    │    │    │    └─ModuleDict: 6-86                                      --\n",
      "│    │    │    └─RobertaIntermediate: 4-20                                       --\n",
      "│    │    │    │    └─LoRALinear: 5-57                                           2,362,368\n",
      "│    │    │    │    │    └─ModuleDict: 6-87                                      --\n",
      "│    │    │    │    └─GELUActivation: 5-58                                       --\n",
      "│    │    │    └─RobertaOutputWithAdapters: 4-21                                 --\n",
      "│    │    │    │    └─LoRALinear: 5-59                                           2,360,064\n",
      "│    │    │    │    │    └─ModuleDict: 6-88                                      --\n",
      "│    │    │    │    └─LayerNorm: 5-60                                            1,536\n",
      "│    │    │    │    └─Dropout: 5-61                                              --\n",
      "│    │    │    │    └─ModuleDict: 5-62                                           --\n",
      "│    │    │    │    └─ModuleDict: 5-63                                           --\n",
      "│    │    └─RobertaLayer: 3-8                                                    --\n",
      "│    │    │    └─RobertaAttention: 4-22                                          --\n",
      "│    │    │    │    └─RobertaSelfAttentionWithAdapters: 5-64                     --\n",
      "│    │    │    │    │    └─LoRALinear: 6-89                                      590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-52                                 --\n",
      "│    │    │    │    │    │    │    └─LoRA: 8-38                                  12,288\n",
      "│    │    │    │    │    └─LoRALinear: 6-90                                      590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-53                                 --\n",
      "│    │    │    │    │    └─LoRALinear: 6-91                                      590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-54                                 --\n",
      "│    │    │    │    │    │    │    └─LoRA: 8-39                                  12,288\n",
      "│    │    │    │    │    └─Dropout: 6-92                                         --\n",
      "│    │    │    │    │    └─PrefixTuningLayer: 6-93                               --\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-55                                 --\n",
      "│    │    │    │    │    │    └─PrefixTuningPool: 7-56                           --\n",
      "│    │    │    │    │    │    │    └─ModuleDict: 8-40                            --\n",
      "│    │    │    │    └─RobertaSelfOutputWithAdapters: 5-65                        --\n",
      "│    │    │    │    │    └─Linear: 6-94                                          590,592\n",
      "│    │    │    │    │    └─LayerNorm: 6-95                                       1,536\n",
      "│    │    │    │    │    └─Dropout: 6-96                                         --\n",
      "│    │    │    │    │    └─ModuleDict: 6-97                                      --\n",
      "│    │    │    │    │    └─ModuleDict: 6-98                                      --\n",
      "│    │    │    └─RobertaIntermediate: 4-23                                       --\n",
      "│    │    │    │    └─LoRALinear: 5-66                                           2,362,368\n",
      "│    │    │    │    │    └─ModuleDict: 6-99                                      --\n",
      "│    │    │    │    └─GELUActivation: 5-67                                       --\n",
      "│    │    │    └─RobertaOutputWithAdapters: 4-24                                 --\n",
      "│    │    │    │    └─LoRALinear: 5-68                                           2,360,064\n",
      "│    │    │    │    │    └─ModuleDict: 6-100                                     --\n",
      "│    │    │    │    └─LayerNorm: 5-69                                            1,536\n",
      "│    │    │    │    └─Dropout: 5-70                                              --\n",
      "│    │    │    │    └─ModuleDict: 5-71                                           --\n",
      "│    │    │    │    └─ModuleDict: 5-72                                           --\n",
      "│    │    └─RobertaLayer: 3-9                                                    --\n",
      "│    │    │    └─RobertaAttention: 4-25                                          --\n",
      "│    │    │    │    └─RobertaSelfAttentionWithAdapters: 5-73                     --\n",
      "│    │    │    │    │    └─LoRALinear: 6-101                                     590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-57                                 --\n",
      "│    │    │    │    │    │    │    └─LoRA: 8-41                                  12,288\n",
      "│    │    │    │    │    └─LoRALinear: 6-102                                     590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-58                                 --\n",
      "│    │    │    │    │    └─LoRALinear: 6-103                                     590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-59                                 --\n",
      "│    │    │    │    │    │    │    └─LoRA: 8-42                                  12,288\n",
      "│    │    │    │    │    └─Dropout: 6-104                                        --\n",
      "│    │    │    │    │    └─PrefixTuningLayer: 6-105                              --\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-60                                 --\n",
      "│    │    │    │    │    │    └─PrefixTuningPool: 7-61                           --\n",
      "│    │    │    │    │    │    │    └─ModuleDict: 8-43                            --\n",
      "│    │    │    │    └─RobertaSelfOutputWithAdapters: 5-74                        --\n",
      "│    │    │    │    │    └─Linear: 6-106                                         590,592\n",
      "│    │    │    │    │    └─LayerNorm: 6-107                                      1,536\n",
      "│    │    │    │    │    └─Dropout: 6-108                                        --\n",
      "│    │    │    │    │    └─ModuleDict: 6-109                                     --\n",
      "│    │    │    │    │    └─ModuleDict: 6-110                                     --\n",
      "│    │    │    └─RobertaIntermediate: 4-26                                       --\n",
      "│    │    │    │    └─LoRALinear: 5-75                                           2,362,368\n",
      "│    │    │    │    │    └─ModuleDict: 6-111                                     --\n",
      "│    │    │    │    └─GELUActivation: 5-76                                       --\n",
      "│    │    │    └─RobertaOutputWithAdapters: 4-27                                 --\n",
      "│    │    │    │    └─LoRALinear: 5-77                                           2,360,064\n",
      "│    │    │    │    │    └─ModuleDict: 6-112                                     --\n",
      "│    │    │    │    └─LayerNorm: 5-78                                            1,536\n",
      "│    │    │    │    └─Dropout: 5-79                                              --\n",
      "│    │    │    │    └─ModuleDict: 5-80                                           --\n",
      "│    │    │    │    └─ModuleDict: 5-81                                           --\n",
      "│    │    └─RobertaLayer: 3-10                                                   --\n",
      "│    │    │    └─RobertaAttention: 4-28                                          --\n",
      "│    │    │    │    └─RobertaSelfAttentionWithAdapters: 5-82                     --\n",
      "│    │    │    │    │    └─LoRALinear: 6-113                                     590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-62                                 --\n",
      "│    │    │    │    │    │    │    └─LoRA: 8-44                                  12,288\n",
      "│    │    │    │    │    └─LoRALinear: 6-114                                     590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-63                                 --\n",
      "│    │    │    │    │    └─LoRALinear: 6-115                                     590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-64                                 --\n",
      "│    │    │    │    │    │    │    └─LoRA: 8-45                                  12,288\n",
      "│    │    │    │    │    └─Dropout: 6-116                                        --\n",
      "│    │    │    │    │    └─PrefixTuningLayer: 6-117                              --\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-65                                 --\n",
      "│    │    │    │    │    │    └─PrefixTuningPool: 7-66                           --\n",
      "│    │    │    │    │    │    │    └─ModuleDict: 8-46                            --\n",
      "│    │    │    │    └─RobertaSelfOutputWithAdapters: 5-83                        --\n",
      "│    │    │    │    │    └─Linear: 6-118                                         590,592\n",
      "│    │    │    │    │    └─LayerNorm: 6-119                                      1,536\n",
      "│    │    │    │    │    └─Dropout: 6-120                                        --\n",
      "│    │    │    │    │    └─ModuleDict: 6-121                                     --\n",
      "│    │    │    │    │    └─ModuleDict: 6-122                                     --\n",
      "│    │    │    └─RobertaIntermediate: 4-29                                       --\n",
      "│    │    │    │    └─LoRALinear: 5-84                                           2,362,368\n",
      "│    │    │    │    │    └─ModuleDict: 6-123                                     --\n",
      "│    │    │    │    └─GELUActivation: 5-85                                       --\n",
      "│    │    │    └─RobertaOutputWithAdapters: 4-30                                 --\n",
      "│    │    │    │    └─LoRALinear: 5-86                                           2,360,064\n",
      "│    │    │    │    │    └─ModuleDict: 6-124                                     --\n",
      "│    │    │    │    └─LayerNorm: 5-87                                            1,536\n",
      "│    │    │    │    └─Dropout: 5-88                                              --\n",
      "│    │    │    │    └─ModuleDict: 5-89                                           --\n",
      "│    │    │    │    └─ModuleDict: 5-90                                           --\n",
      "│    │    └─RobertaLayer: 3-11                                                   --\n",
      "│    │    │    └─RobertaAttention: 4-31                                          --\n",
      "│    │    │    │    └─RobertaSelfAttentionWithAdapters: 5-91                     --\n",
      "│    │    │    │    │    └─LoRALinear: 6-125                                     590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-67                                 --\n",
      "│    │    │    │    │    │    │    └─LoRA: 8-47                                  12,288\n",
      "│    │    │    │    │    └─LoRALinear: 6-126                                     590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-68                                 --\n",
      "│    │    │    │    │    └─LoRALinear: 6-127                                     590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-69                                 --\n",
      "│    │    │    │    │    │    │    └─LoRA: 8-48                                  12,288\n",
      "│    │    │    │    │    └─Dropout: 6-128                                        --\n",
      "│    │    │    │    │    └─PrefixTuningLayer: 6-129                              --\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-70                                 --\n",
      "│    │    │    │    │    │    └─PrefixTuningPool: 7-71                           --\n",
      "│    │    │    │    │    │    │    └─ModuleDict: 8-49                            --\n",
      "│    │    │    │    └─RobertaSelfOutputWithAdapters: 5-92                        --\n",
      "│    │    │    │    │    └─Linear: 6-130                                         590,592\n",
      "│    │    │    │    │    └─LayerNorm: 6-131                                      1,536\n",
      "│    │    │    │    │    └─Dropout: 6-132                                        --\n",
      "│    │    │    │    │    └─ModuleDict: 6-133                                     --\n",
      "│    │    │    │    │    └─ModuleDict: 6-134                                     --\n",
      "│    │    │    └─RobertaIntermediate: 4-32                                       --\n",
      "│    │    │    │    └─LoRALinear: 5-93                                           2,362,368\n",
      "│    │    │    │    │    └─ModuleDict: 6-135                                     --\n",
      "│    │    │    │    └─GELUActivation: 5-94                                       --\n",
      "│    │    │    └─RobertaOutputWithAdapters: 4-33                                 --\n",
      "│    │    │    │    └─LoRALinear: 5-95                                           2,360,064\n",
      "│    │    │    │    │    └─ModuleDict: 6-136                                     --\n",
      "│    │    │    │    └─LayerNorm: 5-96                                            1,536\n",
      "│    │    │    │    └─Dropout: 5-97                                              --\n",
      "│    │    │    │    └─ModuleDict: 5-98                                           --\n",
      "│    │    │    │    └─ModuleDict: 5-99                                           --\n",
      "│    │    └─RobertaLayer: 3-12                                                   --\n",
      "│    │    │    └─RobertaAttention: 4-34                                          --\n",
      "│    │    │    │    └─RobertaSelfAttentionWithAdapters: 5-100                    --\n",
      "│    │    │    │    │    └─LoRALinear: 6-137                                     590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-72                                 --\n",
      "│    │    │    │    │    │    │    └─LoRA: 8-50                                  12,288\n",
      "│    │    │    │    │    └─LoRALinear: 6-138                                     590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-73                                 --\n",
      "│    │    │    │    │    └─LoRALinear: 6-139                                     590,592\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-74                                 --\n",
      "│    │    │    │    │    │    │    └─LoRA: 8-51                                  12,288\n",
      "│    │    │    │    │    └─Dropout: 6-140                                        --\n",
      "│    │    │    │    │    └─PrefixTuningLayer: 6-141                              --\n",
      "│    │    │    │    │    │    └─ModuleDict: 7-75                                 --\n",
      "│    │    │    │    │    │    └─PrefixTuningPool: 7-76                           --\n",
      "│    │    │    │    │    │    │    └─ModuleDict: 8-52                            --\n",
      "│    │    │    │    └─RobertaSelfOutputWithAdapters: 5-101                       --\n",
      "│    │    │    │    │    └─Linear: 6-142                                         590,592\n",
      "│    │    │    │    │    └─LayerNorm: 6-143                                      1,536\n",
      "│    │    │    │    │    └─Dropout: 6-144                                        --\n",
      "│    │    │    │    │    └─ModuleDict: 6-145                                     --\n",
      "│    │    │    │    │    └─ModuleDict: 6-146                                     --\n",
      "│    │    │    └─RobertaIntermediate: 4-35                                       --\n",
      "│    │    │    │    └─LoRALinear: 5-102                                          2,362,368\n",
      "│    │    │    │    │    └─ModuleDict: 6-147                                     --\n",
      "│    │    │    │    └─GELUActivation: 5-103                                      --\n",
      "│    │    │    └─RobertaOutputWithAdapters: 4-36                                 --\n",
      "│    │    │    │    └─LoRALinear: 5-104                                          2,360,064\n",
      "│    │    │    │    │    └─ModuleDict: 6-148                                     --\n",
      "│    │    │    │    └─LayerNorm: 5-105                                           1,536\n",
      "│    │    │    │    └─Dropout: 5-106                                             --\n",
      "│    │    │    │    └─ModuleDict: 5-107                                          --\n",
      "│    │    │    │    └─ModuleDict: 5-108                                          --\n",
      "├─RobertaPooler: 1-3                                                             --\n",
      "│    └─Linear: 2-7                                                               590,592\n",
      "│    └─Tanh: 2-8                                                                 --\n",
      "├─ModuleDict: 1-4                                                                --\n",
      "├─ModuleDict: 1-5                                                                --\n",
      "├─PrefixTuningPool: 1-6                                                          --\n",
      "│    └─ModuleDict: 2-9                                                           --\n",
      "├─PromptTuningLayer: 1-7                                                         38,603,520\n",
      "│    └─Embedding: 2-10                                                           (recursive)\n",
      "│    └─ModuleDict: 2-11                                                          --\n",
      "=========================================================================================================\n",
      "Total params: 164,042,112\n",
      "Trainable params: 164,042,112\n",
      "Non-trainable params: 0\n",
      "=========================================================================================================\n"
     ]
    }
   ],
   "source": [
    "from torchinfo import summary\n",
    "\n",
    "print(str(summary(model.base_model, depth=10)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        ...,\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]])\n"
     ]
    }
   ],
   "source": [
    "# for name in model.state_dict():\n",
    "#   print(name)\n",
    "from torch.nn.parameter import Parameter\n",
    "\n",
    "\n",
    "\n",
    "weights = model.state_dict()['roberta.encoder.layer.0.attention.self.query.loras.rotten_tomatoes.lora_A']\n",
    "print(weights)\n",
    "# 将权重设置为全 0\n",
    "weights.zero_()\n",
    "\n",
    "# 将修改后的权重重新赋值给模型中的对应模块\n",
    "model.roberta.encoder.layer[0].attention.self.query.loras.rotten_tomatoes.lora_A = Parameter(data=weights, requires_grad=False)\n",
    "\n",
    "# # 将模块设置为不可训练\n",
    "# for param in model.roberta.encoder.layer[0].attention.self.query.loras.rotten_tomatoes.lora_A.parameters():\n",
    "#     param.requires_grad = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(174)\n"
     ]
    }
   ],
   "source": [
    "num_small_values = torch.lt(torch.abs(weights), 0.001).sum()\n",
    "print(num_small_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DatasetDict({\n",
      "    test: Dataset({\n",
      "        features: ['sentence1', 'sentence2', 'idx', 'label'],\n",
      "        num_rows: 1104\n",
      "    })\n",
      "})\n",
      "['sentence1', 'sentence2']\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "\n",
    "# 加载数据集\n",
    "dataset = load_dataset(\"super_glue\", 'axb')\n",
    "\n",
    "print(dataset)\n",
    "\n",
    "text_fields = [\n",
    "    field for field in dataset['test'].column_names if field not in ['label', 'idx']]\n",
    "# 遍历每个样本，将所有文本字段的值连接起来，然后保存到新的'text'字段中\n",
    "print(text_fields)\n",
    "\n",
    "inputs = {name: tokenizer(text, return_tensors='pt', padding=True, truncation=True, max_length=512)\n",
    "          for name, text in batch_fields.items()}"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.17 ('cxpt')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  },
  "vscode": {
   "interpreter": {
    "hash": "3162c5d5430b04d3df4a176c558ed82d64364234959ee53c995ae88d96aa71b8"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}