diff --git a/tests/test_general/test_fairness.py b/tests/test_general/test_fairness.py index 859d619..62a05ae 100644 --- a/tests/test_general/test_fairness.py +++ b/tests/test_general/test_fairness.py @@ -1,7 +1,8 @@ import numpy as np import pandas as pd -from sklearn.linear_model import LogisticRegression from sklearn.model_selection import train_test_split +from sklearn.neural_network import MLPClassifier +from sklearn.linear_model import LogisticRegression from stable_gnn.fairness import Fair @@ -71,13 +72,22 @@ def test_fairness(): y_train, y_test, x_train, x_test = train_test_split(y, x, random_state=random_state) cl.fit(y_train, x_train) fairness = Fair(dataset, estimator=cl) - res = fairness.run( - number_iterations=10, - prefit=True, - interior_classifier="knn", - verbose=True, - multiplier=30, - random_state=random_state, - ) - assert (res["accuracy_of_initial_classifier"] - res["accuracy_of_fair_classifier"]) <= 0.05 - assert (res["fairness_of_fair_classifier_diff"]) <= res["fairness_of_initial_classifier_diff"] + accs_fair = [] + accs_init = [] + fairs_fair = [] + fairs_init = [] + for _ in range(15): + res = fairness.run( + number_iterations=10, + prefit=True, + interior_classifier="knn", + verbose=True, + multiplier=30, + random_state=random_state, + ) + accs_fair.append(res["accuracy_of_fair_classifier"]) + accs_init.append(res["accuracy_of_initial_classifier"]) + fairs_fair.append(res["fairness_of_fair_classifier_diff"]) + fairs_init.append(res["fairness_of_initial_classifier_diff"]) + assert (np.mean(accs_init) - np.mean(accs_fair)) <= 0.05 + assert np.mean(fairs_fair) <= np.mean(fairs_init) diff --git a/tutorials/fairness_classification.py b/tutorials/fairness_classification.py new file mode 100644 index 0000000..a3026d9 --- /dev/null +++ b/tutorials/fairness_classification.py @@ -0,0 +1,97 @@ +import numpy as np +import pandas as pd +from sklearn.neural_network import MLPClassifier + +from stable_gnn.fairness import Fair + + +def print_results(res): + accuracy_absolute_loss = res["accuracy_of_initial_classifier"] - res["accuracy_of_fair_classifier"] + accuracy_percentage_loss = accuracy_absolute_loss / res["accuracy_of_initial_classifier"] * 100 + fairness_absolute_improvement = res["fairness_of_initial_classifier_diff"] - res["fairness_of_fair_classifier_diff"] + fairness_percentage_improvement = fairness_absolute_improvement / res["fairness_of_initial_classifier_diff"] * 100 + + f_accuracy = f"Accuracy of initial classifier is {res['accuracy_of_initial_classifier']:0.4f}, while accuracy of fair classifier is \ +{res['accuracy_of_fair_classifier']:0.4f}. Accuracy loss is {accuracy_absolute_loss:0.4f}; it has decreased on \ +{accuracy_percentage_loss:0.4f}%." + + f_fairness = f"Cuae-difference of initial classifier is {res['fairness_of_initial_classifier_diff']:0.4f}, while cuae-difference of fair \ +classifier is {res['fairness_of_fair_classifier_diff']:0.4f}. Fairness improvement is {fairness_absolute_improvement:0.4f}; it has \ +increased on {fairness_percentage_improvement:0.4f}%. " + + print("") + print(f_accuracy) + print("") + print(f_fairness) + return { + "fair_accuracy": res["accuracy_of_fair_classifier"], + "initial_accuracy": res["accuracy_of_initial_classifier"], + "fair_fairness": res["fairness_of_fair_classifier_diff"], + "initial_fairness": res["fairness_of_initial_classifier_diff"], + } + + +def run(name, init_cl): + initial_classifier = init_cl + if name == "LOAN": + loan = pd.read_csv("loan_cleaned.csv") + loan = loan[loan["loan_status"] != "Current"] + + # create risk groups 0 - good, 1 - bad, 2 - dubious + def loan_grouper(x): + if x == "Fully Paid": + z = 0 + elif x == "Charged Off": + z = 1 + elif x == "Late (31-120 days)": + z = 2 + elif x == "Issued": + z = 2 + elif x == "In Grace Period": + z = 2 + elif x == "Late (16-30 days)": + z = 2 + elif x == "Does not meet the credit policy. Status:Fully Paid": + z = 2 + elif x == "Default": + z = 1 + elif x == "Does not meet the credit policy. Status:Charged Off": + z = 1 + return z + + loan["target"] = loan["loan_status"].apply(loan_grouper) + loan = loan[ + [ + "loan_amnt", + "term", + "int_rate", + "verification_status", + "initial_list_status", + "target", + "sub_grade", + "home_ownership", + "purpose", + "dti", + "revol_bal", + "total_pymnt", + "total_rec_prncp", + ] + ] + loan = pd.get_dummies(loan, drop_first=True) + loan = loan.rename(columns={"initial_list_status_w": "attr"}) + + fairness = Fair(dataset=loan, estimator=initial_classifier) + + res = fairness.run(number_iterations=30, interior_classifier="rf", multiplier=20) + + return res + + +if __name__ == "__main__": + accs = [] + fairs = [] + for i in range(10): + dic = run(name="LOAN", init_cl=MLPClassifier(max_iter=300)) + accs.append(dic["accuracy_of_fair_classifier"]) + fairs.append(dic["fairness_of_fair_classifier_diff"]) + print(np.mean(accs), np.std(accs), np.mean(fairs), np.std(fairs)) diff --git a/tutorials/tutorial_fairness.ipynb b/tutorials/tutorial_fairness.ipynb new file mode 100644 index 0000000..390c152 --- /dev/null +++ b/tutorials/tutorial_fairness.ipynb @@ -0,0 +1,414 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "id": "7060b2c9-5270-496a-aac7-dc60a257879e", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from sklearn.neural_network import MLPClassifier\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.metrics import accuracy_score\n", + "\n", + "from stable_gnn.fairness import Fair" + ] + }, + { + "cell_type": "markdown", + "id": "58e233fd-d624-499b-95f6-55f7bde95085", + "metadata": {}, + "source": [ + "# Data preprocesing" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "41784117-19c3-465e-bc13-5bb0bace9c43", + "metadata": {}, + "outputs": [], + "source": [ + "# Load LSAC dataset. We consider race as the sensitive attribute.\n", + "\n", + "d = pd.read_csv('bar_pass_prediction.csv')\n", + "for x in ['ID', 'race1', 'race2', 'sex', 'bar', 'dnn_bar_pass_prediction', 'pass_bar', 'indxgrp2',\n", + " 'gender', 'grad', 'Dropout', 'fulltime', 'lsat', 'zfygpa', 'ugpa', 'zgpa', 'other', 'asian',\n", + " 'black', 'hisp']:\n", + " del d[x]\n", + "\n", + "def grouper_race(x):\n", + " if x == 7:\n", + " return 1\n", + " else: \n", + " return 0\n", + "\n", + "def grouper_gpa(x):\n", + " if x>3.4:\n", + " return 2\n", + " elif x <3.1:\n", + " return 0\n", + " else:\n", + " return 1\n", + "\n", + "d['gpa'] = d['gpa'].apply(grouper_gpa)\n", + "d['race'] = d['race'].apply(grouper_race)\n", + "d = d.rename(columns = {'gpa': 'target', 'race': 'attr'})\n", + "d = pd.get_dummies(d, drop_first=True)\n", + "d = d.dropna(how='any')" + ] + }, + { + "cell_type": "markdown", + "id": "e5e1d5b9-8780-4292-8be1-982465fe4cd9", + "metadata": {}, + "source": [ + "# Fit & predict a decent classifier over data" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "b0d7c1de-5237-4ddd-98a3-14088199599b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[LightGBM] [Warning] Found whitespace in feature_names, replace with underlines\n", + "[LightGBM] [Info] Auto-choosing row-wise multi-threading, the overhead of testing was 0.001186 seconds.\n", + "You can set `force_row_wise=true` to remove the overhead.\n", + "And if memory is not enough, you can set `force_col_wise=true`.\n", + "[LightGBM] [Info] Total Bins 424\n", + "[LightGBM] [Info] Number of data points in the train set: 15320, number of used features: 23\n", + "[LightGBM] [Info] Start training from score -1.110960\n", + "[LightGBM] [Info] Start training from score -1.016623\n", + "[LightGBM] [Info] Start training from score -1.174600\n" + ] + }, + { + "data": { + "text/plain": [ + "0.8519678872136284" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "y = d.drop(['target'], axis=1) \n", + "x = d['target']\n", + "y_train, y_test, x_train, x_test = train_test_split(y, x)\n", + "lg = MLPClassifier()\n", + "lg.fit(y_train, x_train)\n", + "lg_preds = lg.predict(y_test)\n", + "accuracy_score(lg_preds, x_test)" + ] + }, + { + "cell_type": "markdown", + "id": "581cf78b-c143-4cc5-aa33-d1b42a746223", + "metadata": {}, + "source": [ + "# Run FMCLP algorithm " + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "35e4d1a2-99ca-49a4-917b-33906de6d2d7", + "metadata": {}, + "outputs": [], + "source": [ + "# initialize initial classifer and Fair class object.\n", + "initial_classifier = MLPClassifier(verbose=-1)\n", + "fairness = Fair(dataset=d, estimator=initial_classifier)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "51c6a921-b945-4a68-ae31-de823904c170", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 17.4 s, sys: 3.91 s, total: 21.3 s\n", + "Wall time: 18.8 s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "# run FMCLP algorithm\n", + "res = fairness.run(number_iterations=30,\n", + " interior_classifier=\"rf\",\n", + " multiplier=30)" + ] + }, + { + "cell_type": "markdown", + "id": "246ca712-9ebb-463b-821a-b68cc76b83af", + "metadata": {}, + "source": [ + "Before evaluating result we define an auxiliary function to print results. " + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "92297d36-70f2-4a42-896b-a4b0f587d51e", + "metadata": {}, + "outputs": [], + "source": [ + "def print_results(res):\n", + " accuracy_absolute_loss = res['accuracy_of_initial_classifier'] - res['accuracy_of_fair_classifier']\n", + " accuracy_percentage_loss = accuracy_absolute_loss/ res['accuracy_of_initial_classifier']*100\n", + " fairness_absolute_improvement = res['fairness_of_initial_classifier_diff'] - res['fairness_of_fair_classifier_diff'] \n", + " fairness_percentage_improvement = fairness_absolute_improvement/ res['fairness_of_initial_classifier_diff']*100\n", + " \n", + " f_accuracy = f\"Accuracy of initial classifier is {res['accuracy_of_initial_classifier']:0.4f}, while accuracy of fair classifier is \\\n", + "{res['accuracy_of_fair_classifier']:0.4f}. Accuracy loss is {accuracy_absolute_loss:0.4f}; it has decreased on \\\n", + "{accuracy_percentage_loss:0.4f}%.\"\n", + "\n", + " f_fairness = f\"Cuae-difference of initial classifier is {res['fairness_of_initial_classifier_diff']:0.4f}, while cuae-difference of fair \\\n", + "classifier is {res['fairness_of_fair_classifier_diff']:0.4f}. Fairness improvement is {fairness_absolute_improvement:0.4f}; it has \\\n", + "increased on {fairness_percentage_improvement:0.4f}%. \" \n", + "\n", + " print(\"\")\n", + " print(f_accuracy)\n", + " print(\"\")\n", + " print(f_fairness)\n", + " return {'fair_accuracy': res['accuracy_of_fair_classifier'],\n", + " 'initial_accuracy': res['accuracy_of_initial_classifier'],\n", + " 'fair_fairness': res['fairness_of_fair_classifier_diff'] ,\n", + " 'initial_fairness': res['fairness_of_initial_classifier_diff']}" + ] + }, + { + "cell_type": "markdown", + "id": "0cccfbaa-4c7a-4365-96d7-e77f2ffbcdcb", + "metadata": {}, + "source": [ + "Now we evaluate results. We see small loss in accuracy and fairness increase." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "742793d6-9fb6-4fb5-a36b-adc0b5ad5281", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Accuracy of initial classifier is 0.8418, while accuracy of fair classifier is 0.8275. Accuracy loss is 0.0143; it has decreased on 1.6981%.\n", + "\n", + "Cuae-difference of initial classifier is 0.1953, while cuae-difference of fair classifier is 0.1593. Fairness improvement is 0.0361; it has increased on 18.4636%. \n" + ] + }, + { + "data": { + "text/plain": [ + "{'fair_accuracy': 0.8274916780888976,\n", + " 'initial_accuracy': 0.8417857842177403,\n", + " 'fair_fairness': 0.15926052721670658,\n", + " 'initial_fairness': 0.1953244974812931}" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print_results(res)" + ] + }, + { + "cell_type": "markdown", + "id": "b7acfc1c-a2ed-4146-858c-c42e6e7dbd01", + "metadata": {}, + "source": [ + "# Different parameters for FMCLP algorithm." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "2af9befc-b25f-4569-964e-ca929feca1c0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Parameters: interior_classifier rf, multiplier 10\n", + "\n", + "Accuracy of initial classifier is 0.8418, while accuracy of fair classifier is 0.8263. Accuracy loss is 0.0155; it has decreased on 1.8376%.\n", + "\n", + "Cuae-difference of initial classifier is 0.1953, while cuae-difference of fair classifier is 0.1448. Fairness improvement is 0.0505; it has increased on 25.8591%. \n", + "\n", + "\n", + "Parameters: interior_classifier rf, multiplier 20\n", + "\n", + "Accuracy of initial classifier is 0.8418, while accuracy of fair classifier is 0.8281. Accuracy loss is 0.0137; it has decreased on 1.6283%.\n", + "\n", + "Cuae-difference of initial classifier is 0.1953, while cuae-difference of fair classifier is 0.1530. Fairness improvement is 0.0423; it has increased on 21.6486%. \n", + "\n", + "\n", + "Parameters: interior_classifier rf, multiplier 30\n", + "\n", + "Accuracy of initial classifier is 0.8418, while accuracy of fair classifier is 0.8275. Accuracy loss is 0.0143; it has decreased on 1.6981%.\n", + "\n", + "Cuae-difference of initial classifier is 0.1953, while cuae-difference of fair classifier is 0.1593. Fairness improvement is 0.0361; it has increased on 18.4636%. \n", + "\n", + "\n", + "Parameters: interior_classifier rf, multiplier 40\n", + "\n", + "Accuracy of initial classifier is 0.8418, while accuracy of fair classifier is 0.8314. Accuracy loss is 0.0104; it has decreased on 1.2328%.\n", + "\n", + "Cuae-difference of initial classifier is 0.1953, while cuae-difference of fair classifier is 0.1586. Fairness improvement is 0.0367; it has increased on 18.7816%. \n", + "\n", + "\n", + "Parameters: interior_classifier dt, multiplier 10\n", + "\n", + "Accuracy of initial classifier is 0.8418, while accuracy of fair classifier is 0.8148. Accuracy loss is 0.0270; it has decreased on 3.2100%.\n", + "\n", + "Cuae-difference of initial classifier is 0.1953, while cuae-difference of fair classifier is 0.1380. Fairness improvement is 0.0573; it has increased on 29.3589%. \n", + "\n", + "\n", + "Parameters: interior_classifier dt, multiplier 20\n", + "\n", + "Accuracy of initial classifier is 0.8418, while accuracy of fair classifier is 0.8226. Accuracy loss is 0.0192; it has decreased on 2.2796%.\n", + "\n", + "Cuae-difference of initial classifier is 0.1953, while cuae-difference of fair classifier is 0.1497. Fairness improvement is 0.0456; it has increased on 23.3462%. \n", + "\n", + "\n", + "Parameters: interior_classifier dt, multiplier 30\n", + "\n", + "Accuracy of initial classifier is 0.8418, while accuracy of fair classifier is 0.8230. Accuracy loss is 0.0188; it has decreased on 2.2331%.\n", + "\n", + "Cuae-difference of initial classifier is 0.1953, while cuae-difference of fair classifier is 0.1715. Fairness improvement is 0.0238; it has increased on 12.2064%. \n", + "\n", + "\n", + "Parameters: interior_classifier dt, multiplier 40\n", + "\n", + "Accuracy of initial classifier is 0.8418, while accuracy of fair classifier is 0.8238. Accuracy loss is 0.0180; it has decreased on 2.1400%.\n", + "\n", + "Cuae-difference of initial classifier is 0.1953, while cuae-difference of fair classifier is 0.1686. Fairness improvement is 0.0267; it has increased on 13.6887%. \n", + "\n", + "\n", + "Parameters: interior_classifier knn, multiplier 10\n", + "\n", + "Accuracy of initial classifier is 0.8418, while accuracy of fair classifier is 0.8287. Accuracy loss is 0.0131; it has decreased on 1.5585%.\n", + "\n", + "Cuae-difference of initial classifier is 0.1953, while cuae-difference of fair classifier is 0.1393. Fairness improvement is 0.0560; it has increased on 28.6739%. \n", + "\n", + "\n", + "Parameters: interior_classifier knn, multiplier 20\n", + "\n", + "Accuracy of initial classifier is 0.8418, while accuracy of fair classifier is 0.8287. Accuracy loss is 0.0131; it has decreased on 1.5585%.\n", + "\n", + "Cuae-difference of initial classifier is 0.1953, while cuae-difference of fair classifier is 0.1418. Fairness improvement is 0.0535; it has increased on 27.3774%. \n", + "\n", + "\n", + "Parameters: interior_classifier knn, multiplier 30\n", + "\n", + "Accuracy of initial classifier is 0.8418, while accuracy of fair classifier is 0.8291. Accuracy loss is 0.0127; it has decreased on 1.5120%.\n", + "\n", + "Cuae-difference of initial classifier is 0.1953, while cuae-difference of fair classifier is 0.1487. Fairness improvement is 0.0466; it has increased on 23.8745%. \n", + "\n", + "\n", + "Parameters: interior_classifier knn, multiplier 40\n", + "\n", + "Accuracy of initial classifier is 0.8418, while accuracy of fair classifier is 0.8293. Accuracy loss is 0.0125; it has decreased on 1.4887%.\n", + "\n", + "Cuae-difference of initial classifier is 0.1953, while cuae-difference of fair classifier is 0.1508. Fairness improvement is 0.0446; it has increased on 22.8129%. \n", + "\n", + "\n", + "Parameters: interior_classifier lr, multiplier 10\n", + "\n", + "Accuracy of initial classifier is 0.8418, while accuracy of fair classifier is 0.8392. Accuracy loss is 0.0025; it has decreased on 0.3024%.\n", + "\n", + "Cuae-difference of initial classifier is 0.1953, while cuae-difference of fair classifier is 0.1595. Fairness improvement is 0.0359; it has increased on 18.3610%. \n", + "\n", + "\n", + "Parameters: interior_classifier lr, multiplier 20\n", + "\n", + "Accuracy of initial classifier is 0.8418, while accuracy of fair classifier is 0.8375. Accuracy loss is 0.0043; it has decreased on 0.5117%.\n", + "\n", + "Cuae-difference of initial classifier is 0.1953, while cuae-difference of fair classifier is 0.1497. Fairness improvement is 0.0456; it has increased on 23.3462%. \n", + "\n", + "\n", + "Parameters: interior_classifier lr, multiplier 30\n", + "\n", + "Accuracy of initial classifier is 0.8418, while accuracy of fair classifier is 0.8367. Accuracy loss is 0.0051; it has decreased on 0.6048%.\n", + "\n", + "Cuae-difference of initial classifier is 0.1953, while cuae-difference of fair classifier is 0.1487. Fairness improvement is 0.0466; it has increased on 23.8745%. \n", + "\n", + "\n", + "Parameters: interior_classifier lr, multiplier 40\n", + "\n", + "Accuracy of initial classifier is 0.8418, while accuracy of fair classifier is 0.8361. Accuracy loss is 0.0057; it has decreased on 0.6746%.\n", + "\n", + "Cuae-difference of initial classifier is 0.1953, while cuae-difference of fair classifier is 0.1474. Fairness improvement is 0.0479; it has increased on 24.5105%. \n", + "\n", + "\n" + ] + } + ], + "source": [ + "results = []\n", + "for interior_classifier in ['rf', 'dt', 'knn', 'lr']:\n", + " for multiplier in [10, 20, 30, 40]:\n", + " res = fairness.run(number_iterations=30,\n", + " interior_classifier=interior_classifier,\n", + " multiplier=multiplier)\n", + " results.append(res)\n", + " print(f\"Parameters: interior_classifier {interior_classifier}, multiplier {multiplier}\")\n", + " print_results(res)\n", + " print(\"\")\n", + " print(\"\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a9b6c15-7a66-48d7-b47f-9f389bcda90e", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "bamt", + "language": "python", + "name": "bamt" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}