diff --git a/.gitignore b/.gitignore index cf2ccb41..375238bf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *_venv +virny_env notebooks *.env .DS_Store diff --git a/requirements.txt b/requirements.txt index 6e55b6af..12b6aa38 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,6 @@ wheel~=0.38.4 twine~=4.0.2 -requests-toolbelt==1.0.0 numpy~=1.24.2 -datapane~=0.15.5 matplotlib~=3.6.2 pandas~=1.5.2 altair~=4.2.0 @@ -17,4 +15,6 @@ PyYAML~=6.0 river==0.15.0 python-dotenv~=1.0.0 pytest~=7.2.2 -pymongo==4.3.3 \ No newline at end of file +pymongo==4.3.3 +datapane~=0.16.0 +requests-toolbelt==1.0.0 \ No newline at end of file diff --git a/virny/custom_classes/metrics_composer.py b/virny/custom_classes/metrics_composer.py index 4d9cdc10..b18f3866 100644 --- a/virny/custom_classes/metrics_composer.py +++ b/virny/custom_classes/metrics_composer.py @@ -46,19 +46,24 @@ def compose_metrics(self): priv_group = sensitive_attr + '_priv' groups_metrics_dct[sensitive_attr] = { - # Group fairness metrics + # Error disparity metrics 'Equalized_Odds_TPR': cfm[dis_group]['TPR'] - cfm[priv_group]['TPR'], 'Equalized_Odds_FPR': cfm[dis_group]['FPR'] - cfm[priv_group]['FPR'], 'Equalized_Odds_FNR': cfm[dis_group]['FNR'] - cfm[priv_group]['FNR'], 'Disparate_Impact': cfm[dis_group]['Positive-Rate'] / cfm[priv_group]['Positive-Rate'], 'Statistical_Parity_Difference': cfm[dis_group]['Positive-Rate'] - cfm[priv_group]['Positive-Rate'], 'Accuracy_Parity': cfm[dis_group]['Accuracy'] - cfm[priv_group]['Accuracy'], - # Group stability metrics + # Stability disparity metrics 'Label_Stability_Ratio': cfm[dis_group]['Label_Stability'] / cfm[priv_group]['Label_Stability'], 'IQR_Parity': cfm[dis_group]['IQR'] - cfm[priv_group]['IQR'], 'Std_Parity': cfm[dis_group]['Std'] - cfm[priv_group]['Std'], 'Std_Ratio': cfm[dis_group]['Std'] / cfm[priv_group]['Std'], 'Jitter_Parity': cfm[dis_group]['Jitter'] - cfm[priv_group]['Jitter'], + # Uncertainty disparity metrics + 'Overall_Uncertainty_Parity': cfm[dis_group]['Overall_Uncertainty'] - cfm[priv_group]['Overall_Uncertainty'], + 'Overall_Uncertainty_Ratio': cfm[dis_group]['Overall_Uncertainty'] / cfm[priv_group]['Overall_Uncertainty'], + 'Aleatoric_Uncertainty_Parity': cfm[dis_group]['Aleatoric_Uncertainty'] - cfm[priv_group]['Aleatoric_Uncertainty'], + 'Aleatoric_Uncertainty_Ratio': cfm[dis_group]['Aleatoric_Uncertainty'] / cfm[priv_group]['Aleatoric_Uncertainty'], } model_composed_metrics_df = pd.DataFrame(groups_metrics_dct).reset_index()