diff --git a/README.md b/README.md index 7414280a..66119d84 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,6 @@
- ## 📜 Description **Virny** is a Python library for auditing model stability and fairness. The Virny library was diff --git a/docs/examples/Multiple_Models_Interface_Vis.ipynb b/docs/examples/Multiple_Models_Interface_Vis.ipynb new file mode 100644 index 00000000..14fb79b7 --- /dev/null +++ b/docs/examples/Multiple_Models_Interface_Vis.ipynb @@ -0,0 +1,324 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "248cbed8", + "metadata": { + "ExecuteTime": { + "end_time": "2023-09-29T20:56:16.932083Z", + "start_time": "2023-09-29T20:56:16.278169Z" + } + }, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "7ec6cd08", + "metadata": { + "ExecuteTime": { + "end_time": "2023-09-29T20:56:16.940086Z", + "start_time": "2023-09-29T20:56:16.931485Z" + } + }, + "outputs": [], + "source": [ + "import os\n", + "import warnings\n", + "warnings.filterwarnings('ignore')\n", + "os.environ[\"PYTHONWARNINGS\"] = \"ignore\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "b8cb69f2", + "metadata": { + "ExecuteTime": { + "end_time": "2023-09-29T20:56:16.951831Z", + "start_time": "2023-09-29T20:56:16.940588Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Current location: /Users/denys_herasymuk/UCU/4course_2term/Bachelor_Thesis/Code/Virny\n" + ] + } + ], + "source": [ + "cur_folder_name = os.getcwd().split('/')[-1]\n", + "if cur_folder_name != \"Virny\":\n", + " os.chdir(\"../..\")\n", + "\n", + "print('Current location: ', os.getcwd())" + ] + }, + { + "cell_type": "markdown", + "id": "a578f2ab", + "metadata": {}, + "source": [ + "# Multiple Models Interface Usage" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "7a9241de", + "metadata": { + "ExecuteTime": { + "end_time": "2023-09-29T20:56:30.072450Z", + "start_time": "2023-09-29T20:56:22.772584Z" + } + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "from virny.utils.custom_initializers import read_model_metric_dfs, create_config_obj\n", + "from virny.custom_classes.metrics_interactive_visualizer import MetricsInteractiveVisualizer\n", + "from virny.custom_classes.metrics_composer import MetricsComposer" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "outputs": [], + "source": [ + "ROOT_DIR = os.path.join('docs', 'examples')\n", + "config_yaml_path = os.path.join(ROOT_DIR, 'experiment_config.yaml')\n", + "config_yaml_content = \"\"\"\n", + "dataset_name: COMPAS_Without_Sensitive_Attributes\n", + "bootstrap_fraction: 0.8\n", + "n_estimators: 50 # Better to input the higher number of estimators than 100; this is only for this use case example\n", + "sensitive_attributes_dct: {'sex': 1, 'race': 'African-American', 'sex&race': None}\n", + "\"\"\"\n", + "with open(config_yaml_path, 'w', encoding='utf-8') as f:\n", + " f.write(config_yaml_content)\n", + "\n", + "config = create_config_obj(config_yaml_path=config_yaml_path)\n", + "model_names = ['DecisionTreeClassifier', 'LogisticRegression', 'RandomForestClassifier', 'XGBClassifier']\n", + "SAVE_RESULTS_DIR_PATH = os.path.join(ROOT_DIR, 'results', 'COMPAS_Without_Sensitive_Attributes_Metrics_20230812__224136')" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-09-29T20:56:30.095448Z", + "start_time": "2023-09-29T20:56:30.073873Z" + } + }, + "id": "d777610462304f63" + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "f94a20dc", + "metadata": { + "ExecuteTime": { + "end_time": "2023-09-29T20:56:30.121865Z", + "start_time": "2023-09-29T20:56:30.094816Z" + } + }, + "outputs": [], + "source": [ + "models_metrics_dct = read_model_metric_dfs(SAVE_RESULTS_DIR_PATH, model_names=model_names)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "b04d06cf", + "metadata": { + "ExecuteTime": { + "end_time": "2023-09-29T20:56:30.139696Z", + "start_time": "2023-09-29T20:56:30.121071Z" + } + }, + "outputs": [], + "source": [ + "metrics_composer = MetricsComposer(models_metrics_dct, config.sensitive_attributes_dct)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "be6ace22", + "metadata": { + "ExecuteTime": { + "end_time": "2023-09-29T20:56:30.169575Z", + "start_time": "2023-09-29T20:56:30.138633Z" + } + }, + "outputs": [], + "source": [ + "# Compute composed metrics\n", + "models_composed_metrics_df = metrics_composer.compose_metrics()" + ] + }, + { + "cell_type": "code", + "execution_count": 185, + "outputs": [ + { + "data": { + "text/plain": " Metric overall sex_priv sex_priv_correct \\\n0 Mean 0.524270 0.578645 0.600790 \n1 Std 0.067963 0.073618 0.072201 \n2 IQR 0.090596 0.099782 0.098402 \n3 Aleatoric_Uncertainty 0.834874 0.846689 0.826891 \n4 Overall_Uncertainty 0.859083 0.876581 0.856843 \n5 Statistical_Bias 0.405041 0.395811 0.314809 \n6 Jitter 0.106917 0.132090 0.112864 \n7 Per_Sample_Accuracy 0.691061 0.711090 0.918452 \n8 Label_Stability 0.851667 0.807393 0.836903 \n9 TPR 0.679406 0.613333 1.000000 \n10 TNR 0.738462 0.801471 1.000000 \n11 PPV 0.676533 0.630137 1.000000 \n12 FNR 0.320594 0.386667 0.000000 \n13 FPR 0.261538 0.198529 0.000000 \n14 Accuracy 0.712121 0.734597 1.000000 \n15 F1 0.677966 0.621622 1.000000 \n16 Selection-Rate 0.447917 0.345972 0.296774 \n17 Positive-Rate 1.004246 0.973333 1.000000 \n18 Sample_Size 1056.000000 211.000000 155.000000 \n\n sex_priv_incorrect sex_dis sex_dis_correct sex_dis_incorrect \\\n0 0.517352 0.510692 0.514399 0.501767 \n1 0.077539 0.066551 0.064791 0.070788 \n2 0.103600 0.088303 0.085977 0.093900 \n3 0.901488 0.831924 0.817170 0.867440 \n4 0.931213 0.854713 0.839203 0.892051 \n5 0.620012 0.407346 0.301656 0.661771 \n6 0.185306 0.100631 0.091351 0.122972 \n7 0.137143 0.686059 0.936918 0.082177 \n8 0.725714 0.862722 0.873970 0.835645 \n9 0.000000 0.691919 1.000000 0.000000 \n10 0.000000 0.719376 1.000000 0.000000 \n11 0.000000 0.685000 1.000000 0.000000 \n12 1.000000 0.308081 0.000000 1.000000 \n13 1.000000 0.280624 0.000000 1.000000 \n14 0.000000 0.706509 1.000000 0.000000 \n15 0.000000 0.688442 1.000000 0.000000 \n16 0.482143 0.473373 0.458961 0.508065 \n17 0.931034 1.010101 1.000000 1.032787 \n18 56.000000 845.000000 597.000000 248.000000 \n\n race_priv race_priv_correct ... race_dis_correct race_dis_incorrect \\\n0 0.597526 0.618185 ... 0.473863 0.484344 \n1 0.069162 0.066865 ... 0.065947 0.070060 \n2 0.093184 0.089451 ... 0.087919 0.091258 \n3 0.821672 0.807043 ... 0.827404 0.880296 \n4 0.847778 0.832001 ... 0.850193 0.903737 \n5 0.393484 0.296788 ... 0.309510 0.650314 \n6 0.107225 0.097218 ... 0.094812 0.134214 \n7 0.708261 0.930526 ... 0.934866 0.091340 \n8 0.848213 0.861316 ... 0.869732 0.817320 \n9 0.585034 1.000000 ... 1.000000 0.000000 \n10 0.816479 1.000000 ... 1.000000 0.000000 \n11 0.637037 1.000000 ... 1.000000 0.000000 \n12 0.414966 0.000000 ... 0.000000 1.000000 \n13 0.183521 0.000000 ... 0.000000 1.000000 \n14 0.734300 1.000000 ... 1.000000 0.000000 \n15 0.609929 1.000000 ... 1.000000 0.000000 \n16 0.326087 0.282895 ... 0.522321 0.536082 \n17 0.918367 1.000000 ... 1.000000 1.155556 \n18 414.000000 304.000000 ... 448.000000 194.000000 \n\n sex&race_priv sex&race_priv_correct sex&race_priv_incorrect \\\n0 0.586391 0.607290 0.529874 \n1 0.068718 0.066018 0.076019 \n2 0.092020 0.088338 0.101975 \n3 0.832383 0.817398 0.872906 \n4 0.857995 0.841790 0.901818 \n5 0.396398 0.302520 0.650263 \n6 0.108871 0.095304 0.145559 \n7 0.708783 0.933073 0.102254 \n8 0.847224 0.866354 0.795493 \n9 0.595745 1.000000 0.000000 \n10 0.804734 1.000000 0.000000 \n11 0.629213 1.000000 0.000000 \n12 0.404255 0.000000 1.000000 \n13 0.195266 0.000000 1.000000 \n14 0.730038 1.000000 0.000000 \n15 0.612022 1.000000 0.000000 \n16 0.338403 0.291667 0.464789 \n17 0.946809 1.000000 0.868421 \n18 526.000000 384.000000 142.000000 \n\n sex&race_dis sex&race_dis_correct sex&race_dis_incorrect \\\n0 0.462617 0.453857 0.482517 \n1 0.067213 0.066631 0.068536 \n2 0.089184 0.088747 0.090175 \n3 0.837346 0.821026 0.874418 \n4 0.860162 0.843933 0.897027 \n5 0.413620 0.306294 0.657422 \n6 0.104978 0.096287 0.124722 \n7 0.673472 0.933152 0.083580 \n8 0.856075 0.866304 0.832840 \n9 0.734982 1.000000 0.000000 \n10 0.647773 1.000000 0.000000 \n11 0.705085 1.000000 0.000000 \n12 0.265018 0.000000 1.000000 \n13 0.352227 0.000000 1.000000 \n14 0.694340 1.000000 0.000000 \n15 0.719723 1.000000 0.000000 \n16 0.556604 0.565217 0.537037 \n17 1.042403 1.000000 1.160000 \n18 530.000000 368.000000 162.000000 \n\n Model_Name Model_Params \n0 RandomForestClassifier {'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... \n1 RandomForestClassifier {'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... \n2 RandomForestClassifier {'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... \n3 RandomForestClassifier {'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... \n4 RandomForestClassifier {'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... \n5 RandomForestClassifier {'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... \n6 RandomForestClassifier {'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... \n7 RandomForestClassifier {'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... \n8 RandomForestClassifier {'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... \n9 RandomForestClassifier {'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... \n10 RandomForestClassifier {'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... \n11 RandomForestClassifier {'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... \n12 RandomForestClassifier {'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... \n13 RandomForestClassifier {'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... \n14 RandomForestClassifier {'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... \n15 RandomForestClassifier {'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... \n16 RandomForestClassifier {'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... \n17 RandomForestClassifier {'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... \n18 RandomForestClassifier {'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... \n\n[19 rows x 22 columns]", + "text/html": "\n | Metric | \noverall | \nsex_priv | \nsex_priv_correct | \nsex_priv_incorrect | \nsex_dis | \nsex_dis_correct | \nsex_dis_incorrect | \nrace_priv | \nrace_priv_correct | \n... | \nrace_dis_correct | \nrace_dis_incorrect | \nsex&race_priv | \nsex&race_priv_correct | \nsex&race_priv_incorrect | \nsex&race_dis | \nsex&race_dis_correct | \nsex&race_dis_incorrect | \nModel_Name | \nModel_Params | \n
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | \nMean | \n0.524270 | \n0.578645 | \n0.600790 | \n0.517352 | \n0.510692 | \n0.514399 | \n0.501767 | \n0.597526 | \n0.618185 | \n... | \n0.473863 | \n0.484344 | \n0.586391 | \n0.607290 | \n0.529874 | \n0.462617 | \n0.453857 | \n0.482517 | \nRandomForestClassifier | \n{'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... | \n
1 | \nStd | \n0.067963 | \n0.073618 | \n0.072201 | \n0.077539 | \n0.066551 | \n0.064791 | \n0.070788 | \n0.069162 | \n0.066865 | \n... | \n0.065947 | \n0.070060 | \n0.068718 | \n0.066018 | \n0.076019 | \n0.067213 | \n0.066631 | \n0.068536 | \nRandomForestClassifier | \n{'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... | \n
2 | \nIQR | \n0.090596 | \n0.099782 | \n0.098402 | \n0.103600 | \n0.088303 | \n0.085977 | \n0.093900 | \n0.093184 | \n0.089451 | \n... | \n0.087919 | \n0.091258 | \n0.092020 | \n0.088338 | \n0.101975 | \n0.089184 | \n0.088747 | \n0.090175 | \nRandomForestClassifier | \n{'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... | \n
3 | \nAleatoric_Uncertainty | \n0.834874 | \n0.846689 | \n0.826891 | \n0.901488 | \n0.831924 | \n0.817170 | \n0.867440 | \n0.821672 | \n0.807043 | \n... | \n0.827404 | \n0.880296 | \n0.832383 | \n0.817398 | \n0.872906 | \n0.837346 | \n0.821026 | \n0.874418 | \nRandomForestClassifier | \n{'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... | \n
4 | \nOverall_Uncertainty | \n0.859083 | \n0.876581 | \n0.856843 | \n0.931213 | \n0.854713 | \n0.839203 | \n0.892051 | \n0.847778 | \n0.832001 | \n... | \n0.850193 | \n0.903737 | \n0.857995 | \n0.841790 | \n0.901818 | \n0.860162 | \n0.843933 | \n0.897027 | \nRandomForestClassifier | \n{'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... | \n
5 | \nStatistical_Bias | \n0.405041 | \n0.395811 | \n0.314809 | \n0.620012 | \n0.407346 | \n0.301656 | \n0.661771 | \n0.393484 | \n0.296788 | \n... | \n0.309510 | \n0.650314 | \n0.396398 | \n0.302520 | \n0.650263 | \n0.413620 | \n0.306294 | \n0.657422 | \nRandomForestClassifier | \n{'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... | \n
6 | \nJitter | \n0.106917 | \n0.132090 | \n0.112864 | \n0.185306 | \n0.100631 | \n0.091351 | \n0.122972 | \n0.107225 | \n0.097218 | \n... | \n0.094812 | \n0.134214 | \n0.108871 | \n0.095304 | \n0.145559 | \n0.104978 | \n0.096287 | \n0.124722 | \nRandomForestClassifier | \n{'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... | \n
7 | \nPer_Sample_Accuracy | \n0.691061 | \n0.711090 | \n0.918452 | \n0.137143 | \n0.686059 | \n0.936918 | \n0.082177 | \n0.708261 | \n0.930526 | \n... | \n0.934866 | \n0.091340 | \n0.708783 | \n0.933073 | \n0.102254 | \n0.673472 | \n0.933152 | \n0.083580 | \nRandomForestClassifier | \n{'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... | \n
8 | \nLabel_Stability | \n0.851667 | \n0.807393 | \n0.836903 | \n0.725714 | \n0.862722 | \n0.873970 | \n0.835645 | \n0.848213 | \n0.861316 | \n... | \n0.869732 | \n0.817320 | \n0.847224 | \n0.866354 | \n0.795493 | \n0.856075 | \n0.866304 | \n0.832840 | \nRandomForestClassifier | \n{'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... | \n
9 | \nTPR | \n0.679406 | \n0.613333 | \n1.000000 | \n0.000000 | \n0.691919 | \n1.000000 | \n0.000000 | \n0.585034 | \n1.000000 | \n... | \n1.000000 | \n0.000000 | \n0.595745 | \n1.000000 | \n0.000000 | \n0.734982 | \n1.000000 | \n0.000000 | \nRandomForestClassifier | \n{'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... | \n
10 | \nTNR | \n0.738462 | \n0.801471 | \n1.000000 | \n0.000000 | \n0.719376 | \n1.000000 | \n0.000000 | \n0.816479 | \n1.000000 | \n... | \n1.000000 | \n0.000000 | \n0.804734 | \n1.000000 | \n0.000000 | \n0.647773 | \n1.000000 | \n0.000000 | \nRandomForestClassifier | \n{'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... | \n
11 | \nPPV | \n0.676533 | \n0.630137 | \n1.000000 | \n0.000000 | \n0.685000 | \n1.000000 | \n0.000000 | \n0.637037 | \n1.000000 | \n... | \n1.000000 | \n0.000000 | \n0.629213 | \n1.000000 | \n0.000000 | \n0.705085 | \n1.000000 | \n0.000000 | \nRandomForestClassifier | \n{'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... | \n
12 | \nFNR | \n0.320594 | \n0.386667 | \n0.000000 | \n1.000000 | \n0.308081 | \n0.000000 | \n1.000000 | \n0.414966 | \n0.000000 | \n... | \n0.000000 | \n1.000000 | \n0.404255 | \n0.000000 | \n1.000000 | \n0.265018 | \n0.000000 | \n1.000000 | \nRandomForestClassifier | \n{'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... | \n
13 | \nFPR | \n0.261538 | \n0.198529 | \n0.000000 | \n1.000000 | \n0.280624 | \n0.000000 | \n1.000000 | \n0.183521 | \n0.000000 | \n... | \n0.000000 | \n1.000000 | \n0.195266 | \n0.000000 | \n1.000000 | \n0.352227 | \n0.000000 | \n1.000000 | \nRandomForestClassifier | \n{'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... | \n
14 | \nAccuracy | \n0.712121 | \n0.734597 | \n1.000000 | \n0.000000 | \n0.706509 | \n1.000000 | \n0.000000 | \n0.734300 | \n1.000000 | \n... | \n1.000000 | \n0.000000 | \n0.730038 | \n1.000000 | \n0.000000 | \n0.694340 | \n1.000000 | \n0.000000 | \nRandomForestClassifier | \n{'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... | \n
15 | \nF1 | \n0.677966 | \n0.621622 | \n1.000000 | \n0.000000 | \n0.688442 | \n1.000000 | \n0.000000 | \n0.609929 | \n1.000000 | \n... | \n1.000000 | \n0.000000 | \n0.612022 | \n1.000000 | \n0.000000 | \n0.719723 | \n1.000000 | \n0.000000 | \nRandomForestClassifier | \n{'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... | \n
16 | \nSelection-Rate | \n0.447917 | \n0.345972 | \n0.296774 | \n0.482143 | \n0.473373 | \n0.458961 | \n0.508065 | \n0.326087 | \n0.282895 | \n... | \n0.522321 | \n0.536082 | \n0.338403 | \n0.291667 | \n0.464789 | \n0.556604 | \n0.565217 | \n0.537037 | \nRandomForestClassifier | \n{'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... | \n
17 | \nPositive-Rate | \n1.004246 | \n0.973333 | \n1.000000 | \n0.931034 | \n1.010101 | \n1.000000 | \n1.032787 | \n0.918367 | \n1.000000 | \n... | \n1.000000 | \n1.155556 | \n0.946809 | \n1.000000 | \n0.868421 | \n1.042403 | \n1.000000 | \n1.160000 | \nRandomForestClassifier | \n{'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... | \n
18 | \nSample_Size | \n1056.000000 | \n211.000000 | \n155.000000 | \n56.000000 | \n845.000000 | \n597.000000 | \n248.000000 | \n414.000000 | \n304.000000 | \n... | \n448.000000 | \n194.000000 | \n526.000000 | \n384.000000 | \n142.000000 | \n530.000000 | \n368.000000 | \n162.000000 | \nRandomForestClassifier | \n{'bootstrap': True, 'ccp_alpha': 0.0, 'class_w... | \n
19 rows × 22 columns
\n\n | Metric | \nsex | \nrace | \nsex&race | \nModel_Name | \n
---|---|---|---|---|---|
0 | \nEqualized_Odds_TPR | \n0.211919 | \n0.195326 | \n0.183576 | \nDecisionTreeClassifier | \n
1 | \nEqualized_Odds_FPR | \n0.098356 | \n0.104728 | \n0.141078 | \nDecisionTreeClassifier | \n
2 | \nEqualized_Odds_FNR | \n-0.211919 | \n-0.195326 | \n-0.183576 | \nDecisionTreeClassifier | \n
3 | \nDisparate_Impact | \n1.234115 | \n1.135965 | \n1.125105 | \nDecisionTreeClassifier | \n
4 | \nStatistical_Parity_Difference | \n0.193535 | \n0.123016 | \n0.115123 | \nDecisionTreeClassifier | \n
5 | \nAccuracy_Parity | \n0.009832 | \n0.006840 | \n-0.010984 | \nDecisionTreeClassifier | \n
6 | \nLabel_Stability_Ratio | \n1.024740 | \n0.997454 | \n0.995869 | \nDecisionTreeClassifier | \n
7 | \nIQR_Parity | \n0.000768 | \n-0.004804 | \n-0.003282 | \nDecisionTreeClassifier | \n
8 | \nStd_Parity | \n-0.005106 | \n-0.000927 | \n-0.001976 | \nDecisionTreeClassifier | \n
9 | \nStd_Ratio | \n0.931699 | \n0.986984 | \n0.972422 | \nDecisionTreeClassifier | \n
10 | \nJitter_Parity | \n-0.013818 | \n0.007192 | \n0.005364 | \nDecisionTreeClassifier | \n
11 | \nEqualized_Odds_TPR | \n0.166465 | \n0.258440 | \n0.226205 | \nLogisticRegression | \n
12 | \nEqualized_Odds_FPR | \n0.096129 | \n0.156703 | \n0.186079 | \nLogisticRegression | \n
13 | \nEqualized_Odds_FNR | \n-0.166465 | \n-0.258440 | \n-0.226205 | \nLogisticRegression | \n
14 | \nDisparate_Impact | \n1.176075 | \n1.341036 | \n1.263916 | \nLogisticRegression | \n
15 | \nStatistical_Parity_Difference | \n0.145556 | \n0.262157 | \n0.216187 | \nLogisticRegression | \n
16 | \nAccuracy_Parity | \n-0.010286 | \n-0.003747 | \n-0.024119 | \nLogisticRegression | \n
17 | \nLabel_Stability_Ratio | \n1.021988 | \n0.988991 | \n1.003152 | \nLogisticRegression | \n
18 | \nIQR_Parity | \n0.001712 | \n0.001225 | \n0.001058 | \nLogisticRegression | \n
19 | \nStd_Parity | \n0.000822 | \n0.000278 | \n0.000170 | \nLogisticRegression | \n
\n | Metric | \nSEX | \nRAC1P | \nSEX&RAC1P | \nModel_Name | \n
---|---|---|---|---|---|
0 | \nAccuracy_Parity | \n0.047756 | \n0.074977 | \n0.065217 | \nLGBMClassifier__alpha=0.7 | \n
1 | \nAleatoric_Uncertainty_Parity | \n-0.039005 | \n-0.011947 | \n-0.009222 | \nLGBMClassifier__alpha=0.7 | \n
2 | \nAleatoric_Uncertainty_Ratio | \n0.935159 | \n0.979638 | \n0.984220 | \nLGBMClassifier__alpha=0.7 | \n
3 | \nEqualized_Odds_FNR | \n0.030793 | \n-0.110745 | \n-0.052498 | \nLGBMClassifier__alpha=0.7 | \n
4 | \nEqualized_Odds_FPR | \n-0.021317 | \n0.000952 | \n-0.007008 | \nLGBMClassifier__alpha=0.7 | \n
\n | Metric | \nmale | \nrace | \nmale&race | \nModel_Name | \n
---|---|---|---|---|---|
0 | \nAccuracy_Parity | \n-0.024413 | \n-0.158856 | \n-0.162998 | \nLGBMClassifier__alpha=0.6 | \n
1 | \nAleatoric_Uncertainty_Parity | \n-0.016769 | \n0.317464 | \n0.274695 | \nLGBMClassifier__alpha=0.6 | \n
2 | \nAleatoric_Uncertainty_Ratio | \n0.951019 | \n2.126816 | \n1.880052 | \nLGBMClassifier__alpha=0.6 | \n
3 | \nEqualized_Odds_FNR | \n0.006853 | \n0.089260 | \n0.092334 | \nLGBMClassifier__alpha=0.6 | \n
4 | \nEqualized_Odds_FPR | \n0.027311 | \n-0.289259 | \n-0.156572 | \nLGBMClassifier__alpha=0.6 | \n
\n | Metric | \nSEX | \nRAC1P | \nSEX&RAC1P | \nModel_Name | \n
---|---|---|---|---|---|
0 | \nAccuracy_Parity | \n0.026847 | \n0.016299 | \n0.040212 | \nLGBMClassifier__alpha=0.6 | \n
1 | \nAleatoric_Uncertainty_Parity | \n-0.013240 | \n0.027276 | \n0.007235 | \nLGBMClassifier__alpha=0.6 | \n
2 | \nAleatoric_Uncertainty_Ratio | \n0.983584 | \n1.034689 | \n1.009077 | \nLGBMClassifier__alpha=0.6 | \n
3 | \nEqualized_Odds_FNR | \n0.004275 | \n-0.000359 | \n-0.008617 | \nLGBMClassifier__alpha=0.6 | \n
4 | \nEqualized_Odds_FPR | \n-0.012072 | \n-0.024172 | \n-0.040481 | \nLGBMClassifier__alpha=0.6 | \n