Skip to content

Commit

Permalink
Feature/debias wrapper (#152)
Browse files Browse the repository at this point in the history 
Added DebiasWrapper for metrics
  • Loading branch information
@In48semenov
In48semenov authored Aug 5, 2024
1 parent 5793241 commit 4d94d8e
Show file tree
Hide file tree
Showing 13 changed files with 1,008 additions and 79 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## Unreleased

### Added
- `Debias` mechanism for classification, ranking and auc metrics. New parameter `is_debiased` to `calc_from_confusion_df`, `calc_per_user_from_confusion_df` methods of classification metrics, `calc_from_fitted`, `calc_per_user_from_fitted` methods of auc and rankning (`MAP`) metrics, `calc_from_merged`, `calc_per_user_from_merged` methods of ranking (`NDCG`, `MRR`) metrics. ([#152](https://github.com/MobileTeleSystems/RecTools/pull/152))

## [0.7.0] - 29.07.2024

Expand Down
5 changes: 5 additions & 0 deletions rectools/metrics/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,13 @@
`metrics.PairwiseDistanceCalculator`
`metrics.PairwiseHammingDistanceCalculator`
`metrics.SparsePairwiseHammingDistanceCalculator`
`metrics.DebiasConfig`
`metrics.debias_interactions`
"""

from .auc import PAP, PartialAUC
from .classification import MCC, Accuracy, F1Beta, HitRate, Precision, Recall
from .debias import DebiasConfig, debias_interactions
from .distances import (
PairwiseDistanceCalculator,
PairwiseHammingDistanceCalculator,
Expand Down Expand Up @@ -89,4 +92,6 @@
"SufficientReco",
"UnrepeatedReco",
"CoveredUsers",
"DebiasConfig",
"debias_interactions",
)
57 changes: 43 additions & 14 deletions rectools/metrics/auc.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@
from attrs import define, field

from rectools import Columns
from rectools.metrics.base import MetricAtK, outer_merge_reco
from rectools.metrics.base import outer_merge_reco
from rectools.metrics.debias import DebiasableMetrikAtK, calc_debiased_fit_task, debias_interactions


class InsufficientHandling(str, Enum):
Expand Down Expand Up @@ -58,7 +59,7 @@ class AUCFitted:


@define
class _AUCMetric(MetricAtK):
class _AUCMetric(DebiasableMetrikAtK):
"""
ROC AUC based metric base class.
Expand Down Expand Up @@ -88,6 +89,8 @@ class _AUCMetric(MetricAtK):
until the model has non-zero scores for the item in item-item similarity matrix. So with
small `K` for neighbours in ItemKNN and big `K` for `recommend` and AUC based metric you
will still get an error when `insufficient_handling` is set to `raise`.
debias_config : DebiasConfig, optional, default None
Config with debias method parameters (iqr_coef, random_state).
"""

insufficient_handling: str = field(default="ignore")
Expand Down Expand Up @@ -217,36 +220,45 @@ def calc_per_user(self, reco: pd.DataFrame, interactions: pd.DataFrame) -> pd.Se
pd.Series
Values of metric (index - user id, values - metric value for every user).
"""
is_debiased = False
if self.debias_config is not None:
interactions = debias_interactions(interactions, self.debias_config)
is_debiased = True

self._check(reco, interactions=interactions)
insufficient_handling_needed = self.insufficient_handling != InsufficientHandling.IGNORE
fitted = self.fit(reco, interactions, self.k, insufficient_handling_needed)
return self.calc_per_user_from_fitted(fitted)
return self.calc_per_user_from_fitted(fitted, is_debiased)

def calc_from_fitted(self, fitted: AUCFitted) -> float:
def calc_from_fitted(self, fitted: AUCFitted, is_debiased: bool = False) -> float:
"""
Calculate metric value from fitted data.
Parameters
----------
fitted : AUCFitted
Meta data that got from `.fit` method.
is_debiased : bool, default False
An indicator of whether the debias transformation has been applied before or not.
Returns
-------
float
Value of metric (average between users).
"""
per_user = self.calc_per_user_from_fitted(fitted)
per_user = self.calc_per_user_from_fitted(fitted, is_debiased)
return per_user.mean()

def calc_per_user_from_fitted(self, fitted: AUCFitted) -> pd.Series:
def calc_per_user_from_fitted(self, fitted: AUCFitted, is_debiased: bool = False) -> pd.Series:
"""
Calculate metric values for all users from from fitted data.
Parameters
----------
fitted : AUCFitted
Meta data that got from `.fit` method.
is_debiased : bool, default False
An indicator of whether the debias transformation has been applied before or not.
Returns
-------
Expand Down Expand Up @@ -307,6 +319,8 @@ class PartialAUC(_AUCMetric):
until the model has non-zero scores for the item in item-item similarity matrix. So with
small `K` for neighbours in ItemKNN and big `K` for `recommend` and AUC based metric you
will still get an error when `insufficient_handling` is set to `raise`.
debias_config : DebiasConfig, optional, default None
Config with debias method parameters (iqr_coef, random_state).
Examples
--------
Expand Down Expand Up @@ -339,25 +353,26 @@ def _get_sufficient_reco_explanation(self) -> str:
not too high.
"""

def calc_per_user_from_fitted(self, fitted: AUCFitted) -> pd.Series:
def calc_per_user_from_fitted(self, fitted: AUCFitted, is_debiased: bool = False) -> pd.Series:
"""
Calculate metric values for all users from from fitted data.
Parameters
----------
fitted : AUCFitted
Meta data that got from `.fit` method.
is_debiased : bool, default False
An indicator of whether the debias transformation has been applied before or not.
Returns
-------
pd.Series
Values of metric (index - user id, values - metric value for every user).
"""
self._check_debias(is_debiased, obj_name="AUCFitted")
outer_merged = fitted.outer_merged_enriched

# Keep k first false positives for roc auc computation, keep all predicted test positives
cropped = outer_merged[(outer_merged["__fp_cumsum"] < self.k) & (~outer_merged[Columns.Rank].isna())]

cropped_suf, n_pos_suf = self._handle_insufficient_cases(
outer_merged=cropped, n_pos=fitted.n_pos, n_fp_insufficient=fitted.n_fp_insufficient
)
Expand Down Expand Up @@ -415,6 +430,8 @@ class PAP(_AUCMetric):
until the model has non-zero scores for the item in item-item similarity matrix. So with
small `K` for neighbours in ItemKNN and big `K` for `recommend` and AUC based metric you
will still get an error when `insufficient_handling` is set to `raise`.
debias_config : DebiasConfig, optional, default None
Config with debias method parameters (iqr_coef, random_state).
Examples
--------
Expand Down Expand Up @@ -447,22 +464,24 @@ def _get_sufficient_reco_explanation(self) -> str:
for all users.
"""

def calc_per_user_from_fitted(self, fitted: AUCFitted) -> pd.Series:
def calc_per_user_from_fitted(self, fitted: AUCFitted, is_debiased: bool = False) -> pd.Series:
"""
Calculate metric values for all users from outer merged recommendations.
Parameters
----------
fitted : AUCFitted
Meta data that got from `.fit` method.
is_debiased : bool, default False
An indicator of whether the debias transformation has been applied before or not.
Returns
-------
pd.Series
Values of metric (index - user id, values - metric value for every user).
"""
self._check_debias(is_debiased, obj_name="AUCFitted")
outer_merged = fitted.outer_merged_enriched

# Keep k first false positives and k first predicted test positives for roc auc computation
cropped = outer_merged[
(outer_merged["__test_pos_cumsum"] <= self.k)
Expand Down Expand Up @@ -513,12 +532,22 @@ def calc_auc_metrics(
"""
results = {}

k_max = max(metric.k for metric in metrics.values())
insufficient_handling_needed = any(
metric.insufficient_handling != InsufficientHandling.IGNORE for metric in metrics.values()
)
fitted = _AUCMetric.fit(reco, interactions, k_max, insufficient_handling_needed)

debiased_fit_task = calc_debiased_fit_task(metrics.values(), interactions)
fitted_debiased = {}
for debias_config_name, (k_max_d, interactions_d) in debiased_fit_task.items():
fitted_debiased[debias_config_name] = _AUCMetric.fit(
reco, interactions_d, k_max_d, insufficient_handling_needed
)

for name, metric in metrics.items():
results[name] = metric.calc_from_fitted(fitted)
is_debiased = metric.debias_config is not None
results[name] = metric.calc_from_fitted(
fitted=fitted_debiased[metric.debias_config],
is_debiased=is_debiased,
)

return results
Loading

0 comments on commit 4d94d8e

Please sign in to comment.