alma_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6de949f5-550d-46bd-8e39-381e98ba4350",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-09T13:19:39.930812Z",
     "iopub.status.busy": "2024-10-09T13:19:39.930118Z",
     "iopub.status.idle": "2024-10-09T13:19:42.281791Z",
     "shell.execute_reply": "2024-10-09T13:19:42.280997Z",
     "shell.execute_reply.started": "2024-10-09T13:19:39.930772Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%pip install datasets -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f95d6eeb-9776-4453-8789-7f4b502f3abe",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-14T12:44:10.416896Z",
     "iopub.status.busy": "2024-10-14T12:44:10.415939Z",
     "iopub.status.idle": "2024-10-14T12:44:12.082253Z",
     "shell.execute_reply": "2024-10-14T12:44:12.081311Z",
     "shell.execute_reply.started": "2024-10-14T12:44:10.416852Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from datasets import load_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7c9ce272-ee1d-44ff-8d5f-ac8d2c0d5c35",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-10T09:50:06.994944Z",
     "iopub.status.busy": "2024-10-10T09:50:06.994251Z",
     "iopub.status.idle": "2024-10-10T09:50:11.122188Z",
     "shell.execute_reply": "2024-10-10T09:50:11.121296Z",
     "shell.execute_reply.started": "2024-10-10T09:50:06.994905Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "dataset = load_dataset(\"haoranxu/ALMA-R-Preference\", \"ru-en\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d561358a-3329-41a7-a5ff-806873825b62",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-10T09:50:11.124784Z",
     "iopub.status.busy": "2024-10-10T09:50:11.123442Z",
     "iopub.status.idle": "2024-10-10T09:50:11.151715Z",
     "shell.execute_reply": "2024-10-10T09:50:11.150859Z",
     "shell.execute_reply.started": "2024-10-10T09:50:11.124729Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['translation'],\n",
       "        num_rows: 2009\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "606a6e2e-970f-4045-9bdd-897492eec259",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-10T09:50:11.154097Z",
     "iopub.status.busy": "2024-10-10T09:50:11.153387Z",
     "iopub.status.idle": "2024-10-10T09:50:18.906546Z",
     "shell.execute_reply": "2024-10-10T09:50:18.905602Z",
     "shell.execute_reply.started": "2024-10-10T09:50:11.154059Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a6fe594d-c150-4572-b1e9-48f5b5af3b5c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-10T09:50:18.908443Z",
     "iopub.status.busy": "2024-10-10T09:50:18.907702Z",
     "iopub.status.idle": "2024-10-10T09:50:45.204197Z",
     "shell.execute_reply": "2024-10-10T09:50:45.203179Z",
     "shell.execute_reply.started": "2024-10-10T09:50:18.908387Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jupyter/.local/lib/python3.10/site-packages/transformers/utils/hub.py:128: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"cointegrated/roberta-large-cola-krishna2020\")\n",
    "model = AutoModelForSequenceClassification.from_pretrained(\"cointegrated/roberta-large-cola-krishna2020\")\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b1fa562d-8ac7-4c91-9e7a-c1363d9b9b37",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-10T09:50:45.209240Z",
     "iopub.status.busy": "2024-10-10T09:50:45.206075Z",
     "iopub.status.idle": "2024-10-10T09:52:21.018768Z",
     "shell.execute_reply": "2024-10-10T09:52:21.017896Z",
     "shell.execute_reply.started": "2024-10-10T09:50:45.209199Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from datasets import Dataset, DatasetDict\n",
    "\n",
    "new_data = []\n",
    "for item in dataset['train']['translation']:\n",
    "    new_item = item.copy() \n",
    "    \n",
    "    \n",
    "    new_item['gpt4_en_fluency'] = float(torch.softmax(model(**tokenizer(new_item['gpt4_en'], return_tensors=\"pt\").to(device)).logits, dim = -1)[0][0].detach().cpu())\n",
    "    new_item['alma_en_fluency'] = float(torch.softmax(model(**tokenizer(new_item['alma_en'], return_tensors=\"pt\").to(device)).logits, dim = -1)[0][0].detach().cpu())\n",
    "    new_item['ref_en_fluency'] = float(torch.softmax(model(**tokenizer(new_item['en'], return_tensors=\"pt\").to(device)).logits, dim = -1)[0][0].detach().cpu())\n",
    "    \n",
    "    \n",
    "    new_data.append(new_item)\n",
    "    \n",
    "new_dataset = DatasetDict({'train': Dataset.from_dict({'translation': new_data})})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f7a19551-e638-4633-b6bc-7cacfde4668c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-10T09:52:21.020084Z",
     "iopub.status.busy": "2024-10-10T09:52:21.019582Z",
     "iopub.status.idle": "2024-10-10T09:52:21.160284Z",
     "shell.execute_reply": "2024-10-10T09:52:21.159490Z",
     "shell.execute_reply.started": "2024-10-10T09:52:21.020046Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Delta': 0.0,\n",
       " 'alma_en': \"On Monday, researchers at Stanford University's School of Medicine announced the creation of a new diagnostic tool that can sort cells by type; it's a tiny chip that can be printed out using a standard inkjet printer for about a penny.\",\n",
       " 'alma_en_fluency': 0.9950083494186401,\n",
       " 'alma_en_kiwi': 0.8425758481025696,\n",
       " 'alma_en_kiwi_xcomet': 0.8914096057415009,\n",
       " 'alma_en_xcomet': 0.9402433633804321,\n",
       " 'alma_ru': 'В понедельник ученые из Школы медицины Стэнфордского университета объявили о создании нового диагностического инструмента, который может классифицировать клетки по типу: маленькая печатаемая микросхема, которую можно производить с помощью стандартных струйных принтеров по цене примерно в один цент каждая.',\n",
       " 'alma_ru_kiwi': 0.9477968215942383,\n",
       " 'alma_ru_kiwi_xcomet': 0.9473313689231873,\n",
       " 'alma_ru_xcomet': 0.9468659162521362,\n",
       " 'en': 'On Monday, scientists from the Stanford University School of Medicine announced the invention of a new diagnostic tool that can sort cells by type: a tiny printable chip that can be manufactured using standard inkjet printers for possibly about one U.S. cent each.',\n",
       " 'gpt4_en': \"On Monday, scientists from Stanford University School of Medicine announced the invention of a new diagnostic tool that can sort cells by their type; it's a small chip that can be printed using a standard inkjet printer for about 1 cent USD.\",\n",
       " 'gpt4_en_fluency': 0.9940776824951172,\n",
       " 'gpt4_en_kiwi': 0.8634641766548157,\n",
       " 'gpt4_en_kiwi_xcomet': 0.843560665845871,\n",
       " 'gpt4_en_xcomet': 0.8236571550369263,\n",
       " 'gpt4_ru': 'В понедельник учёные из Медицинской школы Стэнфордского университета объявили об изобретении нового диагностического инструмента, который может сортировать клетки по типу: это маленькая печатная микросхема, которую можно производить с использованием стандартных струйных принтеров, возможно, за один цент США каждая.',\n",
       " 'gpt4_ru_kiwi': 0.9254813194274902,\n",
       " 'gpt4_ru_kiwi_xcomet': 0.9345579147338867,\n",
       " 'gpt4_ru_xcomet': 0.9436345100402832,\n",
       " 'language_pair': 'ru-en',\n",
       " 'ref_en_fluency': 0.9918845891952515,\n",
       " 'ref_en_kiwi': 0.8316707015037537,\n",
       " 'ref_en_kiwi_xcomet': 0.8103859126567841,\n",
       " 'ref_en_xcomet': 0.7891011238098145,\n",
       " 'ref_ru_kiwi': 0.941681444644928,\n",
       " 'ref_ru_kiwi_xcomet': 0.9467579424381256,\n",
       " 'ref_ru_xcomet': 0.9518344402313232,\n",
       " 'required_directions': '',\n",
       " 'ru': 'В понедельник ученые из Медицинской школы Стэнфордского университета объявили об изобретении нового диагностического инструмента, который может сортировать клетки по их типу; это маленький чип, который можно напечатать, используя стандартный струйный принтер примерно за 1 цент США.'}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_dataset['train']['translation'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "11ecba8e-db5b-4381-b8de-dc2e30be3676",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-09T13:43:47.534960Z",
     "iopub.status.busy": "2024-10-09T13:43:47.533875Z",
     "iopub.status.idle": "2024-10-09T13:43:49.938604Z",
     "shell.execute_reply": "2024-10-09T13:43:49.937710Z",
     "shell.execute_reply.started": "2024-10-09T13:43:47.534916Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%pip install huggingface_hub -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "007483a6-f6b2-44bd-a632-10f3b87269bf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-10T09:53:46.697328Z",
     "iopub.status.busy": "2024-10-10T09:53:46.696339Z",
     "iopub.status.idle": "2024-10-10T09:53:50.184820Z",
     "shell.execute_reply": "2024-10-10T09:53:50.183853Z",
     "shell.execute_reply.started": "2024-10-10T09:53:46.697286Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Uploading the dataset shards:   0%|          | 0/1 [00:00<?, ?it/s]\n",
      "Creating parquet from Arrow format: 100%|██████████| 3/3 [00:00<00:00, 111.63ba/s]\n",
      "Uploading the dataset shards: 100%|██████████| 1/1 [00:01<00:00,  1.68s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/datasets/marulyanova/ALMA-R-Preference-F/commit/e0b54ddc5efda729a00ee89c8215eb1ed5e14423', commit_message='Upload dataset', commit_description='', oid='e0b54ddc5efda729a00ee89c8215eb1ed5e14423', pr_url=None, repo_url=RepoUrl('https://huggingface.co/datasets/marulyanova/ALMA-R-Preference-F', endpoint='https://huggingface.co', repo_type='dataset', repo_id='marulyanova/ALMA-R-Preference-F'), pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_dataset.push_to_hub(\"marulyanova/ALMA-R-Preference-F\", token = 'your_hf_token')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3c6d17c0-3431-49e5-837c-dfc9f19866f6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-10T09:14:53.892647Z",
     "iopub.status.busy": "2024-10-10T09:14:53.891476Z",
     "iopub.status.idle": "2024-10-10T09:14:54.065706Z",
     "shell.execute_reply": "2024-10-10T09:14:54.064422Z",
     "shell.execute_reply.started": "2024-10-10T09:14:53.892601Z"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'load_dataset' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_2297/1421539369.py\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mnew_dataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"marulyanova/ALMA-R-Preference-F\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'load_dataset' is not defined"
     ]
    }
   ],
   "source": [
    "new_dataset = load_dataset(\"marulyanova/ALMA-R-Preference-F\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6fb551d4-9b5e-4588-a3f4-0e29ecc6a9e0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "DataSphere Kernel",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}