Metrics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "with open( '/share/hel/datasets/jobiqo/talent.com/JobRec/unbalanced_test.pkl', 'rb') as file:\n",
    "    test_dicts = pickle.load(file)\n",
    "bios_test = pd.DataFrame(test_dicts).reset_index()\n",
    "bios_test = bios_test.drop('index', axis=1)\n",
    "\n",
    "\n",
    "with open( '/share/hel/datasets/jobiqo/talent.com/JobRec/uk_jobs.pkl', 'rb') as file:\n",
    "    dicts = pickle.load(file)\n",
    "uk_jobs = pd.DataFrame(dicts).reset_index()\n",
    "uk_jobs = uk_jobs.drop('index', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_path = '/share/rk8/home/deepak/JobAd_Rec/Models/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Avg Neutrality score @10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_neutrality(file):\n",
    "    neutrality_score = 0 \n",
    "    for i in file:\n",
    "        neutrality_score = neutrality_score + uk_jobs.iloc[i['corpus_id'][:10]]['neutrality'].mean()\n",
    "    return neutrality_score/len(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "result = {}\n",
    "for model in os.listdir(base_path):\n",
    "    if 'shahed_result.pkl' in model:\n",
    "        with open(base_path+model, 'rb') as file:\n",
    "            result[model.replace('shahed_result.pkl','')] = get_neutrality(pickle.load(file))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_result = {'0.0':[], '0.1':[], '0.2':[], '0.3':[], '0.4':[],'0.6':[],'0.8':[], '1.0':[]}\n",
    "\n",
    "\n",
    "for key in result.keys():\n",
    "    for new_key in new_result.keys():\n",
    "        if new_key in key:\n",
    "            new_result[new_key].append(result[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('0.0', 0.9531080070241037)\n",
      "('0.1', 0.9546807154010772)\n",
      "('0.2', 0.9544876859445962)\n",
      "('0.3', 0.9575129559801275)\n",
      "('0.4', 0.9561346544970625)\n",
      "('0.6', 0.9576823867542487)\n",
      "('0.8', 0.9613789789099404)\n",
      "('1.0', 0.9775374115320563)\n"
     ]
    }
   ],
   "source": [
    "for key in new_result.keys():\n",
    "    print((key, sum(new_result[key][1:])/len(new_result[key][1:])))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "nDCG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import ndcg_score\n",
    "\n",
    "def get_ndcg(dicts):\n",
    "    ndcg = []\n",
    "    for i in range(len(bios_test)):\n",
    "        ndcg.append(ndcg_score(np.asarray([uk_jobs.iloc[dicts[i]['corpus_id']]['title'].apply(lambda x: 1 if x==bios_test['raw_title'][i] else 0)]),[dicts[i]['scores']],k=10))\n",
    "    \n",
    "    return sum(ndcg)/len(ndcg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline nDCG is 0.745 with std 0.014\n"
     ]
    }
   ],
   "source": [
    "Models = ['1500_distilroberta-base_2024-01-25_02-54-56','1390_distilroberta-base_2024-01-25_02-54-56','5915_distilroberta-base_2024-01-25_02-54-56']\n",
    "\n",
    "result = []\n",
    "\n",
    "for model in Models:\n",
    "    with open(base_path+model+'result.pkl', 'rb') as file:\n",
    "        result.append(get_ndcg(pickle.load(file)))\n",
    "\n",
    "print('Baseline nDCG is {:.3f} with std {:.3f}'.format(np.mean(np.array(result)),np.std(np.array(result))))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline with red words masked during testing gives nDCG is 0.633 with std 0.005\n"
     ]
    }
   ],
   "source": [
    "Models = ['1500_distilroberta-base_2024-01-25_02-54-56','1390_distilroberta-base_2024-01-25_02-54-56','5915_distilroberta-base_2024-01-25_02-54-56']\n",
    "\n",
    "result = []\n",
    "\n",
    "for model in Models:\n",
    "    with open(base_path+model+'mask_result.pkl', 'rb') as file:\n",
    "        result.append(get_ndcg(pickle.load(file)))\n",
    "\n",
    "print('Baseline with red words masked during testing gives nDCG is {:.3f} with std {:.3f}'.format(np.mean(np.array(result)),np.std(np.array(result))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "nDCG separate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.metrics import ndcg_score\n",
    "\n",
    "def get_ndcg_separate(dicts):\n",
    "    ndcg = []\n",
    "    for i in range(len(bios_test)):\n",
    "        ndcg.append(ndcg_score(np.asarray([uk_jobs.iloc[dicts[i]['corpus_id']]['title'].apply(lambda x: 1 if x==bios_test['raw_title'][i] else 0)]),[dicts[i]['scores']],k=10))\n",
    "    \n",
    "\n",
    "    ndcg = [ndcg[i] for i in bios_test[bios_test.gender =='M'].index]\n",
    "    female_ndcg = [ndcg[i] for i in bios_test[bios_test.gender =='F'].index]\n",
    "    \n",
    "    if len(male_ndcg)>0:\n",
    "        male_ndcg =sum(male_ndcg)/len(male_ndcg)\n",
    "    else:\n",
    "        male_ndcg = 0\n",
    "    if len(female_ndcg)>0:\n",
    "        female_ndcg =sum(female_ndcg)/len(female_ndcg)\n",
    "    else:\n",
    "        female_ndcg = 0\n",
    "     \n",
    "    return (male_ndcg, female_ndcg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline male nDCG is 0.737 with std 0.016\n",
      "Baseline female nDCG is 0.752 with std 0.014\n",
      "The gap is 0.015\n"
     ]
    }
   ],
   "source": [
    "Models = ['1500_distilroberta-base_2024-01-25_02-54-56','1390_distilroberta-base_2024-01-25_02-54-56','5915_distilroberta-base_2024-01-25_02-54-56']\n",
    "\n",
    "result_male = []\n",
    "result_female = []\n",
    "\n",
    "for model in Models:\n",
    "    with open(base_path+model+'result.pkl', 'rb') as file:\n",
    "        m,f = get_ndcg_separate(pickle.load(file))\n",
    "        result_male.append(m)\n",
    "        result_female.append(f)\n",
    "\n",
    "\n",
    "\n",
    "print(\"Baseline male nDCG is {:.3f} with std {:.3f}\".format(np.mean(np.array(result_male)),np.std(np.array(result_male))))\n",
    "print(\"Baseline female nDCG is {:.3f} with std {:.3f}\".format(np.mean(np.array(result_female)),np.std(np.array(result_female))))\n",
    "print('The gap is {:.3f}'.format(abs(np.mean(np.array(result_male))-np.mean(np.array(result_female)))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline with red words masked during testing gives male nDCG is 0.642 with std 0.003\n",
      "Baseline with red words masked during testing gives female nDCG is 0.624 with std 0.008\n",
      "The gap is 0.018\n"
     ]
    }
   ],
   "source": [
    "Models = ['1500_distilroberta-base_2024-01-25_02-54-56','1390_distilroberta-base_2024-01-25_02-54-56','5915_distilroberta-base_2024-01-25_02-54-56']\n",
    "\n",
    "result_male = []\n",
    "result_female = []\n",
    "\n",
    "for model in Models:\n",
    "    with open(base_path+model+'mask_result.pkl', 'rb') as file:\n",
    "        m,f = get_ndcg_separate(pickle.load(file))\n",
    "        result_male.append(m)\n",
    "        result_female.append(f)\n",
    "\n",
    "\n",
    "\n",
    "print(\"Baseline with red words masked during testing gives male nDCG is {:.3f} with std {:.3f}\".format(np.mean(np.array(result_male)),np.std(np.array(result_male))))\n",
    "print(\"Baseline with red words masked during testing gives female nDCG is {:.3f} with std {:.3f}\".format(np.mean(np.array(result_female)),np.std(np.array(result_female))))\n",
    "print('The gap is {:.3f}'.format(abs(np.mean(np.array(result_male))-np.mean(np.array(result_female)))))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Testing Counterfactual"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Given the input create counterfactual and get the ranking. You have 2 set of ranking original and couterfactual. Find original vs counterfactual list difference."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/deepak/anaconda3/envs/evonlp/lib/python3.10/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "100%|██████████| 560/560 [1:04:41<00:00,  6.93s/it]\n",
      "100%|██████████| 560/560 [1:06:17<00:00,  7.10s/it]\n",
      "100%|██████████| 560/560 [1:05:11<00:00,  6.99s/it]\n",
      "100%|██████████| 560/560 [1:07:41<00:00,  7.25s/it]\n",
      "100%|██████████| 560/560 [1:02:33<00:00,  6.70s/it]\n",
      "100%|██████████| 560/560 [56:35<00:00,  6.06s/it]\n",
      "100%|██████████| 560/560 [56:34<00:00,  6.06s/it]\n",
      "100%|██████████| 560/560 [56:29<00:00,  6.05s/it]\n",
      "100%|██████████| 560/560 [56:12<00:00,  6.02s/it]\n"
     ]
    }
   ],
   "source": [
    "#Get counterfactual ranking \n",
    "import os\n",
    "from testing import testing\n",
    "\n",
    "device=5\n",
    "pth = '/'\n",
    "\n",
    "base_path = '/share/rk8/home/deepak/JobAd_Rec/Models/'\n",
    "\n",
    "for alpha in ['0.0', '0.1', '0.2', '0.3', '0.4']:\n",
    "    for model in os.listdir(base_path):\n",
    "        if alpha in model and '02-20' in model and 'shahed_result.pkl' not in model:\n",
    "            model_path = base_path+model\n",
    "            try:\n",
    "                testing(path=model_path,gpu=device,counterfactual=True)\n",
    "            except:\n",
    "                pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>raw</th>\n",
       "      <th>raw_title</th>\n",
       "      <th>gender</th>\n",
       "      <th>bio</th>\n",
       "      <th>counter_bio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Klotilda Lala is a staff _ with MRPR and has b...</td>\n",
       "      <td>accountant</td>\n",
       "      <td>F</td>\n",
       "      <td>Alice is a staff _ with MRPR and has been with...</td>\n",
       "      <td>Bob is a staff _ with MRPR and has been with t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Sandy Nicholson is a _ based in Toronto, Canad...</td>\n",
       "      <td>photographer</td>\n",
       "      <td>M</td>\n",
       "      <td>Bob is a _ based in Toronto, Canada. Sandy’s p...</td>\n",
       "      <td>Alice is a _ based in Toronto, Canada. Sandy’s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Susan O'Connell is a Principal Architect for R...</td>\n",
       "      <td>architect</td>\n",
       "      <td>F</td>\n",
       "      <td>Alice is a Principal _ for RevGen Partners, le...</td>\n",
       "      <td>Bob is a Principal _ for RevGen Partners, lead...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Ashley Ritzo is a registered/licensed _ with a...</td>\n",
       "      <td>dietitian</td>\n",
       "      <td>F</td>\n",
       "      <td>Alice is a registered/licensed _ with a B.S. i...</td>\n",
       "      <td>Bob is a registered/licensed _ with a B.S. in ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Jeramie J. Perez is a senior _ in the emerging...</td>\n",
       "      <td>paralegal</td>\n",
       "      <td>M</td>\n",
       "      <td>Bob is a senior _ in the emerging companies pr...</td>\n",
       "      <td>Alice is a senior _ in the emerging companies ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>555</th>\n",
       "      <td>Emily Sayers Slocum is a _-midwife and Doctors...</td>\n",
       "      <td>nurse</td>\n",
       "      <td>F</td>\n",
       "      <td>Alice is a _-midwife and Doctors Without Borde...</td>\n",
       "      <td>Bob is a _-midwife and Doctors Without Borders...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>556</th>\n",
       "      <td>Kara K Sharpf is a Nurse Practitioner Speciali...</td>\n",
       "      <td>nurse</td>\n",
       "      <td>F</td>\n",
       "      <td>Kara K Alice is a _ Practitioner Specialist in...</td>\n",
       "      <td>Kara K Bob is a _ Practitioner Specialist in D...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>557</th>\n",
       "      <td>Shawn Moe is a _ at the IBM lab in Lenexa, Kan...</td>\n",
       "      <td>software engineer</td>\n",
       "      <td>M</td>\n",
       "      <td>Bob is a _ at the IBM lab in Lenexa, Kansas. H...</td>\n",
       "      <td>Alice is a _ at the IBM lab in Lenexa, Kansas....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>558</th>\n",
       "      <td>Mrs. Joscelyn Vanduren is a female Nurse Pract...</td>\n",
       "      <td>nurse</td>\n",
       "      <td>F</td>\n",
       "      <td>Mrs. Alice is a female _ Practitioner (taxonom...</td>\n",
       "      <td>Mrs. Bob is a female _ Practitioner (taxonomy ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>559</th>\n",
       "      <td>Simon Parker is a _ in London. He uses digital...</td>\n",
       "      <td>personal trainer</td>\n",
       "      <td>M</td>\n",
       "      <td>Bob is a _ in London. He uses digital health a...</td>\n",
       "      <td>Alice is a _ in London. she uses digital healt...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>560 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   raw          raw_title  \\\n",
       "0    Klotilda Lala is a staff _ with MRPR and has b...         accountant   \n",
       "1    Sandy Nicholson is a _ based in Toronto, Canad...       photographer   \n",
       "2    Susan O'Connell is a Principal Architect for R...          architect   \n",
       "3    Ashley Ritzo is a registered/licensed _ with a...          dietitian   \n",
       "4    Jeramie J. Perez is a senior _ in the emerging...          paralegal   \n",
       "..                                                 ...                ...   \n",
       "555  Emily Sayers Slocum is a _-midwife and Doctors...              nurse   \n",
       "556  Kara K Sharpf is a Nurse Practitioner Speciali...              nurse   \n",
       "557  Shawn Moe is a _ at the IBM lab in Lenexa, Kan...  software engineer   \n",
       "558  Mrs. Joscelyn Vanduren is a female Nurse Pract...              nurse   \n",
       "559  Simon Parker is a _ in London. He uses digital...   personal trainer   \n",
       "\n",
       "    gender                                                bio  \\\n",
       "0        F  Alice is a staff _ with MRPR and has been with...   \n",
       "1        M  Bob is a _ based in Toronto, Canada. Sandy’s p...   \n",
       "2        F  Alice is a Principal _ for RevGen Partners, le...   \n",
       "3        F  Alice is a registered/licensed _ with a B.S. i...   \n",
       "4        M  Bob is a senior _ in the emerging companies pr...   \n",
       "..     ...                                                ...   \n",
       "555      F  Alice is a _-midwife and Doctors Without Borde...   \n",
       "556      F  Kara K Alice is a _ Practitioner Specialist in...   \n",
       "557      M  Bob is a _ at the IBM lab in Lenexa, Kansas. H...   \n",
       "558      F  Mrs. Alice is a female _ Practitioner (taxonom...   \n",
       "559      M  Bob is a _ in London. He uses digital health a...   \n",
       "\n",
       "                                           counter_bio  \n",
       "0    Bob is a staff _ with MRPR and has been with t...  \n",
       "1    Alice is a _ based in Toronto, Canada. Sandy’s...  \n",
       "2    Bob is a Principal _ for RevGen Partners, lead...  \n",
       "3    Bob is a registered/licensed _ with a B.S. in ...  \n",
       "4    Alice is a senior _ in the emerging companies ...  \n",
       "..                                                 ...  \n",
       "555  Bob is a _-midwife and Doctors Without Borders...  \n",
       "556  Kara K Bob is a _ Practitioner Specialist in D...  \n",
       "557  Alice is a _ at the IBM lab in Lenexa, Kansas....  \n",
       "558  Mrs. Bob is a female _ Practitioner (taxonomy ...  \n",
       "559  Alice is a _ in London. she uses digital healt...  \n",
       "\n",
       "[560 rows x 5 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bios_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "nDCG gap over counterfactual"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.metrics import ndcg_score\n",
    "\n",
    "def get_ndcg_separate(dicts,dicts_counter):\n",
    "    ndcg = []\n",
    "    ndcg_counter = []\n",
    "    for i in range(len(bios_test)):\n",
    "        ndcg.append(ndcg_score(np.asarray([uk_jobs.iloc[dicts[i]['corpus_id']]['title'].apply(lambda x: 1 if x==bios_test['raw_title'][i] else 0)]),[dicts[i]['scores']],k=10))\n",
    "        ndcg_counter.append(ndcg_score(np.asarray([uk_jobs.iloc[dicts_counter[i]['corpus_id']]['title'].apply(lambda x: 1 if x==bios_test['raw_title'][i] else 0)]),[dicts_counter[i]['scores']],k=10))\n",
    "\n",
    "    \n",
    "\n",
    "    ndcg_separation = [abs(ndcg[i]-ndcg_counter[i]) for i in range(len(bios_test))]\n",
    "        \n",
    "    if len(ndcg)>0:\n",
    "        ndcg =sum(ndcg)/len(ndcg)\n",
    "    else:\n",
    "        ndcg = 0\n",
    "    if len(ndcg_counter)>0:\n",
    "        ndcg_counter =sum(ndcg_counter)/len(ndcg_counter)\n",
    "    else:\n",
    "        ndcg_counter = 0\n",
    "    if len(ndcg_separation)>0:\n",
    "        ndcg_separation =sum(ndcg_separation)/len(ndcg_separation)\n",
    "    else:\n",
    "        ndcg_separation = 0 \n",
    "    return (ndcg, ndcg_counter,ndcg_separation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('0.0', 0.026477710728856508, 3)\n",
      "('0.2', 0.035883420193732904, 3)\n",
      "('0.4', 0.05261764709776821, 3)\n",
      "('0.6', 0.02923345778491597, 3)\n",
      "('0.8', 0.012060492937521424, 2)\n",
      "('1.0', 0.00836899053686995, 3)\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "result = {}\n",
    "for model in os.listdir(base_path):\n",
    "    if '02-20' in model and 'shahed_result.pkl' in model:\n",
    "        model_path = base_path+model\n",
    "        \n",
    "        with open( model_path,'rb') as file1:\n",
    "            \n",
    "            with open( model_path.replace('shahed','counter'),'rb') as file2:\n",
    "                result[model.replace('shahed_result.pkl','')] = get_ndcg_separate(pickle.load(file1),pickle.load(file2))[-1]\n",
    "\n",
    "\n",
    "new_result = {'0.0':[], '0.2':[], '0.4':[],'0.6':[],'0.8':[], '1.0':[]}\n",
    "\n",
    "\n",
    "for key in result.keys():\n",
    "    for new_key in new_result.keys():\n",
    "        if new_key in key:\n",
    "            new_result[new_key].append(result[key])\n",
    "\n",
    "for key in new_result.keys():\n",
    "    print((key, sum(new_result[key])/len(new_result[key]),len(new_result[key])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.026477710728856508\n",
      "0.035883420193732904\n",
      "0.05261764709776821\n",
      "0.02923345778491597\n",
      "0.012060492937521424\n",
      "0.00836899053686995\n"
     ]
    }
   ],
   "source": [
    "for key in new_result.keys():\n",
    "    print(sum(new_result[key])/len(new_result[key]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SDR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def SDR(dicts):\n",
    "    male_item_ids=[]\n",
    "    female_item_ids=[]\n",
    "    for male in [dicts[i] for i in bios_test[bios_test.gender =='M'].index]:\n",
    "        male_df = pd.DataFrame(male)\n",
    "        male_item_ids = male_item_ids+list(male_df[male_df['scores']>sorted(male_df['scores'],reverse=True)[10]]['corpus_id'])\n",
    "    \n",
    "    for female in [dicts[i] for i in bios_test[bios_test.gender =='F'].index]:\n",
    "        female_df = pd.DataFrame(female)\n",
    "        female_item_ids = female_item_ids+list(female_df[female_df['scores']>sorted(female_df['scores'],reverse=True)[10]]['corpus_id'])\n",
    "\n",
    "    output = (len(male_item_ids)+len(female_item_ids)-2*len(set(male_item_ids).intersection(female_item_ids)))/(len(male_item_ids)+len(female_item_ids)-len(set(male_item_ids).intersection(female_item_ids)))\n",
    "    #output = len(set(male_item_ids).intersection(female_item_ids))\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "result = {}\n",
    "for model in os.listdir(base_path):\n",
    "    if '02-20' in model and 'shahed_result.pkl' in model:\n",
    "        with open(base_path+model, 'rb') as file:\n",
    "            result[model.replace('shahed_result.pkl','')] = SDR(pickle.load(file))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_result = {'0.0':[], '0.2':[], '0.4':[],'0.6':[],'0.8':[], '1.0':[]}\n",
    "\n",
    "\n",
    "for key in result.keys():\n",
    "    for new_key in new_result.keys():\n",
    "        if new_key in key:\n",
    "            new_result[new_key].append(result[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('0.0', 0.9098421606837759)\n",
      "('0.2', 0.9109633927218549)\n",
      "('0.4', 0.9116280081505629)\n",
      "('0.6', 0.929333368271465)\n",
      "('0.8', 0.9795598465644189)\n",
      "('1.0', 0.9828790894340358)\n"
     ]
    }
   ],
   "source": [
    "for key in new_result.keys():\n",
    "    print((key, sum(new_result[key])/len(new_result[key])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LDR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.metrics import ndcg_score\n",
    "\n",
    "def LDR(dicts,dicts_counter):\n",
    "    output = []\n",
    "    \n",
    "    for i in range(len(bios_test)):\n",
    "        output.append((sum(((dicts[i]['corpus_id']!=dicts_counter[i]['corpus_id'])*1)[:10]))/10)\n",
    "        #print(((dicts[i]['corpus_id']!=dicts_counter[i]['corpus_id'])*1)[:10])\n",
    "        #break\n",
    "           \n",
    "    if len(output)>0:\n",
    "        ldr =sum(output)/len(output)\n",
    "    else:\n",
    "        ldr = 0\n",
    "     \n",
    "    return ldr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('0.0', 0.6904166666666663, 3)\n",
      "('0.2', 0.6778571428571422, 3)\n",
      "('0.4', 0.6302976190476185, 3)\n",
      "('0.6', 0.48666666666666664, 3)\n",
      "('0.8', 0.5874107142857145, 2)\n",
      "('1.0', 0.42988095238095236, 3)\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "result = {}\n",
    "for model in os.listdir(base_path):\n",
    "    if '02-20' in model and 'shahed_result.pkl' in model:\n",
    "        model_path = base_path+model\n",
    "        \n",
    "        with open( model_path,'rb') as file1:\n",
    "            \n",
    "            with open( model_path.replace('shahed','counter'),'rb') as file2:\n",
    "                result[model.replace('shahed_result.pkl','')] = LDR(pickle.load(file1),pickle.load(file2))\n",
    "\n",
    "\n",
    "new_result = {'0.0':[], '0.2':[], '0.4':[],'0.6':[],'0.8':[], '1.0':[]}\n",
    "\n",
    "\n",
    "for key in result.keys():\n",
    "    for new_key in new_result.keys():\n",
    "        if new_key in key:\n",
    "            new_result[new_key].append(result[key])\n",
    "\n",
    "for key in new_result.keys():\n",
    "    print((key, sum(new_result[key])/len(new_result[key]),len(new_result[key])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6904166666666663\n",
      "0.6778571428571422\n",
      "0.6302976190476185\n",
      "0.48666666666666664\n",
      "0.5874107142857145\n",
      "0.42988095238095236\n"
     ]
    }
   ],
   "source": [
    "for key in new_result.keys():\n",
    "    print( sum(new_result[key])/len(new_result[key]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Re-Ranking functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.metrics import ndcg_score\n",
    "from sklearn import preprocessing\n",
    "\n",
    "import pandas as pd\n",
    "from sklearn import preprocessing\n",
    "\n",
    "\n",
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "\n",
    "\n",
    "def get_ndcg_separate(dicts):\n",
    "    ndcg = []\n",
    "    for i in range(len(bios_test)):\n",
    "        ndcg.append(ndcg_score(np.asarray([uk_jobs.iloc[dicts[i]['corpus_id']]['title'].apply(lambda x: 1 if x==bios_test['raw_title'][i] else 0)]),[dicts[i]['scores']],k=10))\n",
    "    \n",
    "\n",
    "    male_ndcg = [ndcg[i] for i in bios_test[bios_test.gender =='M'].index]\n",
    "    female_ndcg = [ndcg[i] for i in bios_test[bios_test.gender =='F'].index]\n",
    "    \n",
    "    if len(male_ndcg)>0:\n",
    "        male_ndcg =sum(male_ndcg)/len(male_ndcg)\n",
    "    else:\n",
    "        male_ndcg = 0\n",
    "    if len(female_ndcg)>0:\n",
    "        female_ndcg =sum(female_ndcg)/len(female_ndcg)\n",
    "    else:\n",
    "        female_ndcg = 0\n",
    "     \n",
    "    return (male_ndcg, female_ndcg)\n",
    "\n",
    "from sklearn.metrics import ndcg_score\n",
    "\n",
    "def get_ndcg(dicts):\n",
    "    ndcg = []\n",
    "    for i in range(len(bios_test)):\n",
    "        ndcg.append(ndcg_score(np.asarray([uk_jobs.iloc[dicts[i]['corpus_id']]['title'].apply(lambda x: 1 if x==bios_test['raw_title'][i] else 0)]),[dicts[i]['scores']],k=10))\n",
    "    \n",
    "    return sum(ndcg)/len(ndcg)\n",
    "\n",
    "\n",
    "\n",
    "def averageing_func(row, scores_mean, scores_std,neut_mean, neut_std):\n",
    "    lam=0.5\n",
    "    if scores_std == 0:\n",
    "        scores_std =1\n",
    "    if neut_std == 0:\n",
    "        neut_std = 1\n",
    "    return (1-lam)*(row['scores']-scores_mean)*(1/scores_std)+lam*(row['neutrality']-neut_mean)*(1/neut_std)\n",
    "\n",
    "def get_reranked_ndcg(dicts,top=50):\n",
    "    ndcg = []\n",
    "\n",
    "    for i in range(len(bios_test)):\n",
    "        new_uk_jobs= uk_jobs.iloc[dicts[i]['corpus_id'][:top]].copy()\n",
    "        new_uk_jobs['scores'] = dicts[i]['scores'][:top]\n",
    "        scores_mean, scores_std = new_uk_jobs['scores'].mean(),new_uk_jobs['scores'].std()\n",
    "        neut_mean, neut_std = new_uk_jobs['neutrality'].mean(),new_uk_jobs['neutrality'].std()\n",
    "        new_uk_jobs['sorting'] = new_uk_jobs.apply(averageing_func, args=(scores_mean, scores_std,neut_mean, neut_std), axis=1)\n",
    "        neutrality = new_uk_jobs.sort_values(by='sorting',ascending=False)\n",
    "        ndcg.append(ndcg_score(np.asarray([neutrality['title'].apply(lambda x: 1 if x==bios_test['raw_title'][i] else 0)]),[neutrality['scores']],k=10))\n",
    "\n",
    "    return sum(ndcg)/len(ndcg)\n",
    "\n",
    "def new_get_ndcg_separate(dicts,lam,top=10):\n",
    "    ndcg = []\n",
    "    for i in range(len(bios_test)):\n",
    "        new_uk_jobs= uk_jobs.iloc[dicts[i]['corpus_id'][:top]].copy()\n",
    "        new_uk_jobs['scores'] = dicts[i]['scores'][:top]\n",
    "        scores_mean, scores_std = new_uk_jobs['scores'].mean(),new_uk_jobs['scores'].std()\n",
    "        neut_mean, neut_std = new_uk_jobs['neutrality'].mean(),new_uk_jobs['neutrality'].std()\n",
    "        new_uk_jobs['sorting'] = new_uk_jobs.apply(averageing_func, args=(scores_mean, scores_std,neut_mean, neut_std), axis=1)\n",
    "        neutrality = new_uk_jobs.sort_values(by='sorting',ascending=False)\n",
    "        ndcg.append(ndcg_score(np.asarray([neutrality['title'].apply(lambda x: 1 if x==bios_test['raw_title'][i] else 0)]),[neutrality['scores']],k=10))\n",
    "    \n",
    "\n",
    "    male_ndcg = [ndcg[i] for i in bios_test[bios_test.gender =='M'].index]\n",
    "    female_ndcg = [ndcg[i] for i in bios_test[bios_test.gender =='F'].index]\n",
    "    \n",
    "    if len(male_ndcg)>0:\n",
    "        male_ndcg =sum(male_ndcg)/len(male_ndcg)\n",
    "    else:\n",
    "        male_ndcg = 0\n",
    "    if len(female_ndcg)>0:\n",
    "        female_ndcg =sum(female_ndcg)/len(female_ndcg)\n",
    "    else:\n",
    "        female_ndcg = 0\n",
    "     \n",
    "    return (male_ndcg, female_ndcg)\n",
    "\n",
    "\n",
    "def new_LDR(dicts,dicts_counter,top=50):\n",
    "    output = []\n",
    "    \n",
    "    for i in range(len(bios_test)):\n",
    "        new_uk_jobs= uk_jobs.iloc[dicts[i]['corpus_id'][:top]].copy()\n",
    "        new_uk_jobs['scores'] = dicts[i]['scores'][:top]\n",
    "        scores_mean, scores_std = new_uk_jobs['scores'].mean(),new_uk_jobs['scores'].std()\n",
    "        neut_mean, neut_std = new_uk_jobs['neutrality'].mean(),new_uk_jobs['neutrality'].std()\n",
    "        new_uk_jobs['sorting'] = new_uk_jobs.apply(averageing_func, args=(scores_mean, scores_std,neut_mean, neut_std), axis=1)\n",
    "        neutrality = new_uk_jobs.sort_values(by='sorting',ascending=False)\n",
    "        new_uk_jobs= uk_jobs.iloc[dicts_counter[i]['corpus_id'][:top]].copy()\n",
    "        new_uk_jobs['scores'] = dicts_counter[i]['scores'][:top]\n",
    "        scores_mean, scores_std = new_uk_jobs['scores'].mean(),new_uk_jobs['scores'].std()\n",
    "        neut_mean, neut_std = new_uk_jobs['neutrality'].mean(),new_uk_jobs['neutrality'].std()\n",
    "        new_uk_jobs['sorting'] = new_uk_jobs.apply(averageing_func, args=(scores_mean, scores_std,neut_mean, neut_std), axis=1)\n",
    "        neutrality_counter = new_uk_jobs.sort_values(by='sorting',ascending=False)\n",
    "        output.append((sum(((neutrality.index!=neutrality_counter.index)*1)[:10]))/10)\n",
    "           \n",
    "    if len(output)>0:\n",
    "        ldr =sum(output)/len(output)\n",
    "    else:\n",
    "        ldr = 0\n",
    "     \n",
    "    return ldr\n",
    "\n",
    "\n",
    "def LDR(dicts,dicts_counter):\n",
    "    output = []\n",
    "    \n",
    "    for i in range(len(bios_test)):\n",
    "        output.append((sum(((dicts[i]['corpus_id']!=dicts_counter[i]['corpus_id'])*1)[:10]))/10)\n",
    "           \n",
    "    if len(output)>0:\n",
    "        ldr =sum(output)/len(output)\n",
    "    else:\n",
    "        ldr = 0\n",
    "     \n",
    "    return ldr\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "evonlp",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}