Elo_Ranking.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "578a8ed8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "import json, math\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "pd.options.display.float_format = '{:.2f}'.format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7d5031bb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_a</th>\n",
       "      <th>model_b</th>\n",
       "      <th>winner</th>\n",
       "      <th>judge</th>\n",
       "      <th>turn</th>\n",
       "      <th>anony</th>\n",
       "      <th>language</th>\n",
       "      <th>tstamp</th>\n",
       "      <th>source</th>\n",
       "      <th>data</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>koala-13b</td>\n",
       "      <td>gpt-4</td>\n",
       "      <td>model_b</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706445022.31</td>\n",
       "      <td>koala-13b-vs-gpt-4.jsonl</td>\n",
       "      <td>{'instruction': 'Kuka keksi psykiatrian', 'ins...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gpt-4</td>\n",
       "      <td>koala-13b</td>\n",
       "      <td>model_a</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706445022.31</td>\n",
       "      <td>koala-13b-vs-gpt-4.jsonl</td>\n",
       "      <td>{'instruction': 'Implementing an RNN with atte...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>koala-13b</td>\n",
       "      <td>gpt-4</td>\n",
       "      <td>model_b</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706445022.31</td>\n",
       "      <td>koala-13b-vs-gpt-4.jsonl</td>\n",
       "      <td>{'instruction': 'Write a sample function in Py...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gpt-4</td>\n",
       "      <td>koala-13b</td>\n",
       "      <td>model_a</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706445022.31</td>\n",
       "      <td>koala-13b-vs-gpt-4.jsonl</td>\n",
       "      <td>{'instruction': 'Ile to 2 razy 2?', 'instructi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>gpt-4</td>\n",
       "      <td>koala-13b</td>\n",
       "      <td>model_a</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706445022.31</td>\n",
       "      <td>koala-13b-vs-gpt-4.jsonl</td>\n",
       "      <td>{'instruction': 'Generate a sentence using zoo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7466</th>\n",
       "      <td>vicuna-13b</td>\n",
       "      <td>koala-13b</td>\n",
       "      <td>tie</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706445025.73</td>\n",
       "      <td>vicuna-13b-vs-koala-13b.jsonl</td>\n",
       "      <td>{'instruction': 'Do not include any non-numeri...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7467</th>\n",
       "      <td>vicuna-13b</td>\n",
       "      <td>koala-13b</td>\n",
       "      <td>tie</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706445025.73</td>\n",
       "      <td>vicuna-13b-vs-koala-13b.jsonl</td>\n",
       "      <td>{'instruction': 'Write the letters in sequence...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7468</th>\n",
       "      <td>vicuna-13b</td>\n",
       "      <td>koala-13b</td>\n",
       "      <td>model_b</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706445025.73</td>\n",
       "      <td>vicuna-13b-vs-koala-13b.jsonl</td>\n",
       "      <td>{'instruction': 'Hello', 'instruction_trans': ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7469</th>\n",
       "      <td>koala-13b</td>\n",
       "      <td>vicuna-13b</td>\n",
       "      <td>model_b</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706445025.73</td>\n",
       "      <td>vicuna-13b-vs-koala-13b.jsonl</td>\n",
       "      <td>{'instruction': 'Hi', 'instruction_trans': '',...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7470</th>\n",
       "      <td>vicuna-13b</td>\n",
       "      <td>koala-13b</td>\n",
       "      <td>tie</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706445025.73</td>\n",
       "      <td>vicuna-13b-vs-koala-13b.jsonl</td>\n",
       "      <td>{'instruction': 'Who was the president of the ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7471 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         model_a     model_b   winner       judge  turn  anony language   \n",
       "0      koala-13b       gpt-4  model_b  arena_user     1   True  English  \\\n",
       "1          gpt-4   koala-13b  model_a  arena_user     1   True  English   \n",
       "2      koala-13b       gpt-4  model_b  arena_user     1   True  English   \n",
       "3          gpt-4   koala-13b  model_a  arena_user     1   True  English   \n",
       "4          gpt-4   koala-13b  model_a  arena_user     1   True  English   \n",
       "...          ...         ...      ...         ...   ...    ...      ...   \n",
       "7466  vicuna-13b   koala-13b      tie  arena_user     1   True  English   \n",
       "7467  vicuna-13b   koala-13b      tie  arena_user     1   True  English   \n",
       "7468  vicuna-13b   koala-13b  model_b  arena_user     1   True  English   \n",
       "7469   koala-13b  vicuna-13b  model_b  arena_user     1   True  English   \n",
       "7470  vicuna-13b   koala-13b      tie  arena_user     1   True  English   \n",
       "\n",
       "            tstamp                         source   \n",
       "0    1706445022.31       koala-13b-vs-gpt-4.jsonl  \\\n",
       "1    1706445022.31       koala-13b-vs-gpt-4.jsonl   \n",
       "2    1706445022.31       koala-13b-vs-gpt-4.jsonl   \n",
       "3    1706445022.31       koala-13b-vs-gpt-4.jsonl   \n",
       "4    1706445022.31       koala-13b-vs-gpt-4.jsonl   \n",
       "...            ...                            ...   \n",
       "7466 1706445025.73  vicuna-13b-vs-koala-13b.jsonl   \n",
       "7467 1706445025.73  vicuna-13b-vs-koala-13b.jsonl   \n",
       "7468 1706445025.73  vicuna-13b-vs-koala-13b.jsonl   \n",
       "7469 1706445025.73  vicuna-13b-vs-koala-13b.jsonl   \n",
       "7470 1706445025.73  vicuna-13b-vs-koala-13b.jsonl   \n",
       "\n",
       "                                                   data  \n",
       "0     {'instruction': 'Kuka keksi psykiatrian', 'ins...  \n",
       "1     {'instruction': 'Implementing an RNN with atte...  \n",
       "2     {'instruction': 'Write a sample function in Py...  \n",
       "3     {'instruction': 'Ile to 2 razy 2?', 'instructi...  \n",
       "4     {'instruction': 'Generate a sentence using zoo...  \n",
       "...                                                 ...  \n",
       "7466  {'instruction': 'Do not include any non-numeri...  \n",
       "7467  {'instruction': 'Write the letters in sequence...  \n",
       "7468  {'instruction': 'Hello', 'instruction_trans': ...  \n",
       "7469  {'instruction': 'Hi', 'instruction_trans': '',...  \n",
       "7470  {'instruction': 'Who was the president of the ...  \n",
       "\n",
       "[7471 rows x 10 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filename = './data/examples/chatbot_arena-annotation.json'\n",
    "raw_data = pd.read_json(filename).sort_values(ascending=True, by=[\"tstamp\"])\n",
    "raw_data.loc[raw_data['winner'] == 'modedl_b', 'winner'] = 'model_b'\n",
    "\n",
    "battles = raw_data[raw_data['anony']].reset_index(drop=True)\n",
    "battles"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88910cac",
   "metadata": {},
   "source": [
    "MAD selected data, using `text-embedding-ada-002` metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0e688f1b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_a</th>\n",
       "      <th>model_b</th>\n",
       "      <th>winner</th>\n",
       "      <th>judge</th>\n",
       "      <th>turn</th>\n",
       "      <th>anony</th>\n",
       "      <th>language</th>\n",
       "      <th>tstamp</th>\n",
       "      <th>source</th>\n",
       "      <th>data</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gpt-4</td>\n",
       "      <td>koala-13b</td>\n",
       "      <td>model_a</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706515434.84</td>\n",
       "      <td>koala-13b-vs-gpt-4.jsonl</td>\n",
       "      <td>{'instruction': 'Write a sample function in Py...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>koala-13b</td>\n",
       "      <td>gpt-4</td>\n",
       "      <td>model_b</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706515434.84</td>\n",
       "      <td>koala-13b-vs-gpt-4.jsonl</td>\n",
       "      <td>{'instruction': 'Generate a sentence using zoo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gpt-4</td>\n",
       "      <td>koala-13b</td>\n",
       "      <td>tie</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706515434.84</td>\n",
       "      <td>koala-13b-vs-gpt-4.jsonl</td>\n",
       "      <td>{'instruction': 'What is the most efficient wa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>koala-13b</td>\n",
       "      <td>gpt-4</td>\n",
       "      <td>tie</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706515434.84</td>\n",
       "      <td>koala-13b-vs-gpt-4.jsonl</td>\n",
       "      <td>{'instruction': 'Write a hilarious original jo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>gpt-4</td>\n",
       "      <td>koala-13b</td>\n",
       "      <td>tie</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706515434.84</td>\n",
       "      <td>koala-13b-vs-gpt-4.jsonl</td>\n",
       "      <td>{'instruction': 'faça uma fanfic +18 em ptbr',...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205</th>\n",
       "      <td>koala-13b</td>\n",
       "      <td>vicuna-13b</td>\n",
       "      <td>model_b</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706515434.97</td>\n",
       "      <td>vicuna-13b-vs-koala-13b.jsonl</td>\n",
       "      <td>{'instruction': 'Half of my heart is in Havana...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206</th>\n",
       "      <td>vicuna-13b</td>\n",
       "      <td>koala-13b</td>\n",
       "      <td>model_a</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706515434.97</td>\n",
       "      <td>vicuna-13b-vs-koala-13b.jsonl</td>\n",
       "      <td>{'instruction': 'Fill in the blanks: my name i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207</th>\n",
       "      <td>koala-13b</td>\n",
       "      <td>vicuna-13b</td>\n",
       "      <td>tie</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706515434.97</td>\n",
       "      <td>vicuna-13b-vs-koala-13b.jsonl</td>\n",
       "      <td>{'instruction': 'у Барсика конфета у Мурзика п...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>koala-13b</td>\n",
       "      <td>vicuna-13b</td>\n",
       "      <td>tie</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706515434.97</td>\n",
       "      <td>vicuna-13b-vs-koala-13b.jsonl</td>\n",
       "      <td>{'instruction': 'Write me a poem in urdu in th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209</th>\n",
       "      <td>koala-13b</td>\n",
       "      <td>vicuna-13b</td>\n",
       "      <td>tie</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706515434.97</td>\n",
       "      <td>vicuna-13b-vs-koala-13b.jsonl</td>\n",
       "      <td>{'instruction': 'Beende folgenden Satz: eene m...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>210 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        model_a     model_b   winner       judge  turn  anony language   \n",
       "0         gpt-4   koala-13b  model_a  arena_user     1   True  English  \\\n",
       "1     koala-13b       gpt-4  model_b  arena_user     1   True  English   \n",
       "2         gpt-4   koala-13b      tie  arena_user     1   True  English   \n",
       "3     koala-13b       gpt-4      tie  arena_user     1   True  English   \n",
       "4         gpt-4   koala-13b      tie  arena_user     1   True  English   \n",
       "..          ...         ...      ...         ...   ...    ...      ...   \n",
       "205   koala-13b  vicuna-13b  model_b  arena_user     1   True  English   \n",
       "206  vicuna-13b   koala-13b  model_a  arena_user     1   True  English   \n",
       "207   koala-13b  vicuna-13b      tie  arena_user     1   True  English   \n",
       "208   koala-13b  vicuna-13b      tie  arena_user     1   True  English   \n",
       "209   koala-13b  vicuna-13b      tie  arena_user     1   True  English   \n",
       "\n",
       "           tstamp                         source   \n",
       "0   1706515434.84       koala-13b-vs-gpt-4.jsonl  \\\n",
       "1   1706515434.84       koala-13b-vs-gpt-4.jsonl   \n",
       "2   1706515434.84       koala-13b-vs-gpt-4.jsonl   \n",
       "3   1706515434.84       koala-13b-vs-gpt-4.jsonl   \n",
       "4   1706515434.84       koala-13b-vs-gpt-4.jsonl   \n",
       "..            ...                            ...   \n",
       "205 1706515434.97  vicuna-13b-vs-koala-13b.jsonl   \n",
       "206 1706515434.97  vicuna-13b-vs-koala-13b.jsonl   \n",
       "207 1706515434.97  vicuna-13b-vs-koala-13b.jsonl   \n",
       "208 1706515434.97  vicuna-13b-vs-koala-13b.jsonl   \n",
       "209 1706515434.97  vicuna-13b-vs-koala-13b.jsonl   \n",
       "\n",
       "                                                  data  \n",
       "0    {'instruction': 'Write a sample function in Py...  \n",
       "1    {'instruction': 'Generate a sentence using zoo...  \n",
       "2    {'instruction': 'What is the most efficient wa...  \n",
       "3    {'instruction': 'Write a hilarious original jo...  \n",
       "4    {'instruction': 'faça uma fanfic +18 em ptbr',...  \n",
       "..                                                 ...  \n",
       "205  {'instruction': 'Half of my heart is in Havana...  \n",
       "206  {'instruction': 'Fill in the blanks: my name i...  \n",
       "207  {'instruction': 'у Барсика конфета у Мурзика п...  \n",
       "208  {'instruction': 'Write me a poem in urdu in th...  \n",
       "209  {'instruction': 'Beende folgenden Satz: eene m...  \n",
       "\n",
       "[210 rows x 10 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filename = './data/examples/chatbot_arena-annotation-10-embedding.json'\n",
    "raw_data = pd.read_json(filename).sort_values(ascending=True, by=[\"tstamp\"])\n",
    "raw_data.loc[raw_data['winner'] == 'modedl_b', 'winner'] = 'model_b'\n",
    "\n",
    "battles_10_embedding = raw_data[raw_data['anony']].reset_index(drop=True)\n",
    "battles_10_embedding"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "425e05c0",
   "metadata": {},
   "source": [
    "MAD selected data, using `bert-score` metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9058b6f7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_a</th>\n",
       "      <th>model_b</th>\n",
       "      <th>winner</th>\n",
       "      <th>judge</th>\n",
       "      <th>turn</th>\n",
       "      <th>anony</th>\n",
       "      <th>language</th>\n",
       "      <th>tstamp</th>\n",
       "      <th>source</th>\n",
       "      <th>data</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>koala-13b</td>\n",
       "      <td>gpt-4</td>\n",
       "      <td>model_b</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706513835.23</td>\n",
       "      <td>koala-13b-vs-gpt-4.jsonl</td>\n",
       "      <td>{'instruction': '- 너는 여러가지 신용카드를 파는 신용카드 판매원이야...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>koala-13b</td>\n",
       "      <td>gpt-4</td>\n",
       "      <td>model_b</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706513835.23</td>\n",
       "      <td>koala-13b-vs-gpt-4.jsonl</td>\n",
       "      <td>{'instruction': '안녕?!', 'instruction_trans': '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>koala-13b</td>\n",
       "      <td>gpt-4</td>\n",
       "      <td>model_b</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706513835.23</td>\n",
       "      <td>koala-13b-vs-gpt-4.jsonl</td>\n",
       "      <td>{'instruction': 'Hãy làm một bài thơ về phong ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gpt-4</td>\n",
       "      <td>koala-13b</td>\n",
       "      <td>tie</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706513835.23</td>\n",
       "      <td>koala-13b-vs-gpt-4.jsonl</td>\n",
       "      <td>{'instruction': 'ты говоришь по русски?', 'ins...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>koala-13b</td>\n",
       "      <td>gpt-4</td>\n",
       "      <td>model_b</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706513835.23</td>\n",
       "      <td>koala-13b-vs-gpt-4.jsonl</td>\n",
       "      <td>{'instruction': 'Cześć! Umiesz polski?', 'inst...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205</th>\n",
       "      <td>koala-13b</td>\n",
       "      <td>vicuna-13b</td>\n",
       "      <td>tie</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706513835.36</td>\n",
       "      <td>vicuna-13b-vs-koala-13b.jsonl</td>\n",
       "      <td>{'instruction': 'Beende folgenden Satz: eene m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206</th>\n",
       "      <td>koala-13b</td>\n",
       "      <td>vicuna-13b</td>\n",
       "      <td>model_b</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706513835.36</td>\n",
       "      <td>vicuna-13b-vs-koala-13b.jsonl</td>\n",
       "      <td>{'instruction': 'Напиши стих или поэму или про...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207</th>\n",
       "      <td>vicuna-13b</td>\n",
       "      <td>koala-13b</td>\n",
       "      <td>tie</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706513835.36</td>\n",
       "      <td>vicuna-13b-vs-koala-13b.jsonl</td>\n",
       "      <td>{'instruction': 'Xavier 's mom has 3 kids, Gro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>vicuna-13b</td>\n",
       "      <td>koala-13b</td>\n",
       "      <td>tie</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706513835.36</td>\n",
       "      <td>vicuna-13b-vs-koala-13b.jsonl</td>\n",
       "      <td>{'instruction': 'output a random json string a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209</th>\n",
       "      <td>koala-13b</td>\n",
       "      <td>vicuna-13b</td>\n",
       "      <td>tie</td>\n",
       "      <td>arena_user</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>English</td>\n",
       "      <td>1706513835.36</td>\n",
       "      <td>vicuna-13b-vs-koala-13b.jsonl</td>\n",
       "      <td>{'instruction': 'Write me a poem in urdu in th...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>210 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        model_a     model_b   winner       judge  turn  anony language   \n",
       "0     koala-13b       gpt-4  model_b  arena_user     1   True  English  \\\n",
       "1     koala-13b       gpt-4  model_b  arena_user     1   True  English   \n",
       "2     koala-13b       gpt-4  model_b  arena_user     1   True  English   \n",
       "3         gpt-4   koala-13b      tie  arena_user     1   True  English   \n",
       "4     koala-13b       gpt-4  model_b  arena_user     1   True  English   \n",
       "..          ...         ...      ...         ...   ...    ...      ...   \n",
       "205   koala-13b  vicuna-13b      tie  arena_user     1   True  English   \n",
       "206   koala-13b  vicuna-13b  model_b  arena_user     1   True  English   \n",
       "207  vicuna-13b   koala-13b      tie  arena_user     1   True  English   \n",
       "208  vicuna-13b   koala-13b      tie  arena_user     1   True  English   \n",
       "209   koala-13b  vicuna-13b      tie  arena_user     1   True  English   \n",
       "\n",
       "           tstamp                         source   \n",
       "0   1706513835.23       koala-13b-vs-gpt-4.jsonl  \\\n",
       "1   1706513835.23       koala-13b-vs-gpt-4.jsonl   \n",
       "2   1706513835.23       koala-13b-vs-gpt-4.jsonl   \n",
       "3   1706513835.23       koala-13b-vs-gpt-4.jsonl   \n",
       "4   1706513835.23       koala-13b-vs-gpt-4.jsonl   \n",
       "..            ...                            ...   \n",
       "205 1706513835.36  vicuna-13b-vs-koala-13b.jsonl   \n",
       "206 1706513835.36  vicuna-13b-vs-koala-13b.jsonl   \n",
       "207 1706513835.36  vicuna-13b-vs-koala-13b.jsonl   \n",
       "208 1706513835.36  vicuna-13b-vs-koala-13b.jsonl   \n",
       "209 1706513835.36  vicuna-13b-vs-koala-13b.jsonl   \n",
       "\n",
       "                                                  data  \n",
       "0    {'instruction': '- 너는 여러가지 신용카드를 파는 신용카드 판매원이야...  \n",
       "1    {'instruction': '안녕?!', 'instruction_trans': '...  \n",
       "2    {'instruction': 'Hãy làm một bài thơ về phong ...  \n",
       "3    {'instruction': 'ты говоришь по русски?', 'ins...  \n",
       "4    {'instruction': 'Cześć! Umiesz polski?', 'inst...  \n",
       "..                                                 ...  \n",
       "205  {'instruction': 'Beende folgenden Satz: eene m...  \n",
       "206  {'instruction': 'Напиши стих или поэму или про...  \n",
       "207  {'instruction': 'Xavier 's mom has 3 kids, Gro...  \n",
       "208  {'instruction': 'output a random json string a...  \n",
       "209  {'instruction': 'Write me a poem in urdu in th...  \n",
       "\n",
       "[210 rows x 10 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filename = './data/examples/chatbot_arena-annotation-10-bert.json'\n",
    "raw_data = pd.read_json(filename).sort_values(ascending=True, by=[\"tstamp\"])\n",
    "raw_data.loc[raw_data['winner'] == 'modedl_b', 'winner'] = 'model_b'\n",
    "\n",
    "battles_10_bert = raw_data[raw_data['anony']].reset_index(drop=True)\n",
    "battles_10_bert"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "825ab7a5",
   "metadata": {},
   "source": [
    "# Elo Rating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "368dfe84",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_elo(battles, K=4, SCALE=400, BASE=10, INIT_RATING=1000):\n",
    "    rating = defaultdict(lambda: INIT_RATING)\n",
    "\n",
    "    for rd, model_a, model_b, winner in battles[['model_a', 'model_b', 'winner']].itertuples():\n",
    "        ra = rating[model_a]\n",
    "        rb = rating[model_b]\n",
    "        ea = 1 / (1 + BASE ** ((rb - ra) / SCALE))\n",
    "        eb = 1 / (1 + BASE ** ((ra - rb) / SCALE))\n",
    "        if winner == \"model_a\":\n",
    "            sa = 1\n",
    "        elif winner == \"model_b\":\n",
    "            sa = 0\n",
    "        elif winner == \"tie\" or winner == \"tie (bothbad)\":\n",
    "            sa = 0.5\n",
    "        else:\n",
    "            raise Exception(f\"unexpected vote {winner}\")\n",
    "        rating[model_a] += K * (sa - ea)\n",
    "        rating[model_b] += K * (1 - sa - eb)\n",
    "\n",
    "    return rating\n",
    "\n",
    "def preety_print_elo_ratings(ratings):\n",
    "    df = pd.DataFrame([\n",
    "        [n, elo_ratings[n]] for n in elo_ratings.keys()\n",
    "    ], columns=[\"Model\", \"Elo rating\"]).sort_values(\"Elo rating\", ascending=False).reset_index(drop=True)\n",
    "    df[\"Elo rating\"] = (df[\"Elo rating\"] + 0.5).astype(int)\n",
    "    df.index = df.index + 1\n",
    "    return df\n",
    "\n",
    "def get_bootstrap_result(battles, func_compute_elo, num_round):\n",
    "    rows = []\n",
    "    for i in tqdm(range(num_round), desc=\"bootstrap\"):\n",
    "        rows.append(func_compute_elo(battles.sample(frac=1.0, replace=False)))\n",
    "    df = pd.DataFrame(rows)\n",
    "    return df[df.median().sort_values(ascending=False).index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d3413c4b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "bootstrap: 100%|██████████████████████████████████████████████████████████████████| 1000/1000 [00:09<00:00, 104.62it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>Elo rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gpt-4</td>\n",
       "      <td>1192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>claude-v1</td>\n",
       "      <td>1133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gpt-3.5-turbo</td>\n",
       "      <td>1073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>vicuna-13b</td>\n",
       "      <td>996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>koala-13b</td>\n",
       "      <td>935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>alpaca-13b</td>\n",
       "      <td>848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>chatglm-6b</td>\n",
       "      <td>824</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           model  Elo rating\n",
       "0          gpt-4        1192\n",
       "1      claude-v1        1133\n",
       "2  gpt-3.5-turbo        1073\n",
       "3     vicuna-13b         996\n",
       "4      koala-13b         935\n",
       "5     alpaca-13b         848\n",
       "6     chatglm-6b         824"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BOOTSTRAP_ROUNDS = 1000\n",
    "\n",
    "np.random.seed(42)\n",
    "bootstrap_elo_lu = get_bootstrap_result(battles, compute_elo, BOOTSTRAP_ROUNDS)\n",
    "bootstrap_lu_median = bootstrap_elo_lu.median().reset_index().set_axis([\"model\", \"Elo rating\"], axis=1)\n",
    "bootstrap_lu_median[\"Elo rating\"] = (bootstrap_lu_median[\"Elo rating\"] + 0.5).astype(int)\n",
    "bootstrap_lu_median"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9f9c08b7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "bootstrap: 100%|█████████████████████████████████████████████████████████████████| 1000/1000 [00:00<00:00, 1168.08it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>Elo rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gpt-4</td>\n",
       "      <td>1048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gpt-3.5-turbo</td>\n",
       "      <td>1012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>claude-v1</td>\n",
       "      <td>1008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>vicuna-13b</td>\n",
       "      <td>1002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>koala-13b</td>\n",
       "      <td>987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>alpaca-13b</td>\n",
       "      <td>982</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>chatglm-6b</td>\n",
       "      <td>962</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           model  Elo rating\n",
       "0          gpt-4        1048\n",
       "1  gpt-3.5-turbo        1012\n",
       "2      claude-v1        1008\n",
       "3     vicuna-13b        1002\n",
       "4      koala-13b         987\n",
       "5     alpaca-13b         982\n",
       "6     chatglm-6b         962"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# text-embedding-ada-002\n",
    "\n",
    "BOOTSTRAP_ROUNDS = 1000\n",
    "\n",
    "np.random.seed(42)\n",
    "bootstrap_elo_lu = get_bootstrap_result(battles_10_embedding, compute_elo, BOOTSTRAP_ROUNDS)\n",
    "bootstrap_lu_median = bootstrap_elo_lu.median().reset_index().set_axis([\"model\", \"Elo rating\"], axis=1)\n",
    "bootstrap_lu_median[\"Elo rating\"] = (bootstrap_lu_median[\"Elo rating\"] + 0.5).astype(int)\n",
    "bootstrap_lu_median"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "642ec16f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "bootstrap: 100%|█████████████████████████████████████████████████████████████████| 1000/1000 [00:00<00:00, 1446.79it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>Elo rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gpt-4</td>\n",
       "      <td>1056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>claude-v1</td>\n",
       "      <td>1038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gpt-3.5-turbo</td>\n",
       "      <td>1020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>vicuna-13b</td>\n",
       "      <td>1002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>koala-13b</td>\n",
       "      <td>967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>alpaca-13b</td>\n",
       "      <td>964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>chatglm-6b</td>\n",
       "      <td>954</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           model  Elo rating\n",
       "0          gpt-4        1056\n",
       "1      claude-v1        1038\n",
       "2  gpt-3.5-turbo        1020\n",
       "3     vicuna-13b        1002\n",
       "4      koala-13b         967\n",
       "5     alpaca-13b         964\n",
       "6     chatglm-6b         954"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# bert-score\n",
    "BOOTSTRAP_ROUNDS = 1000\n",
    "\n",
    "np.random.seed(42)\n",
    "bootstrap_elo_lu = get_bootstrap_result(battles_10_bert, compute_elo, BOOTSTRAP_ROUNDS)\n",
    "bootstrap_lu_median = bootstrap_elo_lu.median().reset_index().set_axis([\"model\", \"Elo rating\"], axis=1)\n",
    "bootstrap_lu_median[\"Elo rating\"] = (bootstrap_lu_median[\"Elo rating\"] + 0.5).astype(int)\n",
    "bootstrap_lu_median"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "openai",
   "language": "python",
   "name": "openai"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}