From 0e4392656cc3be84a6c6b80b7b249c89502c922b Mon Sep 17 00:00:00 2001 From: "bbbeomjin@gmail.com" Date: Fri, 2 Aug 2024 16:17:05 +0900 Subject: [PATCH 1/2] Added the pseudo quantile Huber loss --- mqboost/base.py | 1 + mqboost/objective.py | 41 ++++++++++++++++++++++++++++++++++++----- mqboost/regressor.py | 2 +- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/mqboost/base.py b/mqboost/base.py index a40012e..1dfc355 100644 --- a/mqboost/base.py +++ b/mqboost/base.py @@ -39,6 +39,7 @@ class ModelName(BaseEnum): class ObjectiveName(BaseEnum): check: str = "check" huber: str = "huber" + phuber: str = "phuber" class TypeName(BaseEnum): diff --git a/mqboost/objective.py b/mqboost/objective.py index 33f3c9a..8095251 100644 --- a/mqboost/objective.py +++ b/mqboost/objective.py @@ -21,6 +21,11 @@ def _grad_rho(u: np.ndarray, alpha: float) -> np.ndarray: return (u < 0).astype(int) - alpha +def _hess_rho(u: np.ndarray, alpha: float) -> np.ndarray: + _h = np.ones_like(u) + return _h + + def _rho(u: np.ndarray, alpha: float) -> np.ndarray: """ Compute the check loss function. @@ -62,6 +67,23 @@ def _grad_huber(u: np.ndarray, alpha: float, delta: float) -> np.ndarray: return _r * _smaller_delta + _g * _bigger_delta +def _hess_huber(u: np.ndarray, alpha: float, delta: float) -> np.ndarray: + _h = np.ones_like(u) + return _h + + +def _grad_phuber(u: np.ndarray, alpha: float, delta: float) -> np.ndarray: + scale = delta**2 + u**2 + _g = -abs(_grad_rho(u, alpha)) * u / scale**(1/2) + return _g + + +def _hess_phuber(u: np.ndarray, alpha: float, delta: float) -> np.ndarray: + scale = 1 + (u / delta)**2 + _h = (1 / delta) * abs(_grad_rho(u, alpha)) / (scale**(3/2)) + return _h + + def _train_pred_reshape( y_pred: np.ndarray, dtrain: DtrainLike, @@ -85,6 +107,7 @@ def _compute_grads_hess( dtrain: DtrainLike, alphas: List[float], grad_fn: Callable[[np.ndarray, float, Any], np.ndarray], + hess_fn: Callable[[np.ndarray, float, Any], np.ndarray], **kwargs: Any, ) -> Tuple[np.ndarray, np.ndarray]: """ @@ -94,6 +117,7 @@ def _compute_grads_hess( dtrain (DtrainLike): The training data. alphas (List[float]): List of quantile levels. grad_fn (Callable): The gradient function to be used. + hess_fn (Callable): The Hessian function to be used. **kwargs (Any): Additional arguments for the gradient function. Returns: @@ -103,16 +127,20 @@ def _compute_grads_hess( _y_train, _y_pred = _train_pred_reshape( y_pred=y_pred, dtrain=dtrain, len_alpha=_len_alpha ) - grads = [] + grads = []; hess = [] for alpha_inx in range(len(alphas)): _err_for_alpha = _y_train[alpha_inx] - _y_pred[alpha_inx] - _grad = grad_fn(u=_err_for_alpha, alpha=alphas[alpha_inx], **kwargs) + _grad = grad_fn(u = _err_for_alpha, alpha = alphas[alpha_inx], **kwargs) + _hess = hess_fn(u = _err_for_alpha, alpha = alphas[alpha_inx], **kwargs) grads.append(_grad) - return np.concatenate(grads), np.ones_like(y_pred) + hess.append(_hess) + + return np.concatenate(grads), np.concatenate(hess) -check_loss_grad_hess: Callable = partial(_compute_grads_hess, grad_fn=_grad_rho) -huber_loss_grad_hess: Callable = partial(_compute_grads_hess, grad_fn=_grad_huber) +check_loss_grad_hess: Callable = partial(_compute_grads_hess, grad_fn=_grad_rho, hess_fn = _hess_rho) +huber_loss_grad_hess: Callable = partial(_compute_grads_hess, grad_fn=_grad_huber, hess_fn = _hess_huber) +phuber_loss_grad_hess: Callable = partial(_compute_grads_hess, grad_fn=_grad_phuber, hess_fn = _hess_phuber) def _eval_check_loss( @@ -209,6 +237,9 @@ def __init__( self._fobj = partial(huber_loss_grad_hess, alphas=alphas, delta=self._delta) elif objective == ObjectiveName.check: self._fobj = partial(check_loss_grad_hess, alphas=alphas) + elif objective == ObjectiveName.phuber: + self._delta = delta_validate(delta = delta) + self._fobj = partial(phuber_loss_grad_hess, alphas = alphas, delta = self._delta) self._eval_name = CHECK_LOSS if model == ModelName.lightgbm: diff --git a/mqboost/regressor.py b/mqboost/regressor.py index 49374f4..1588956 100644 --- a/mqboost/regressor.py +++ b/mqboost/regressor.py @@ -42,7 +42,7 @@ def __init__( params: Dict[str, Any], model: str = ModelName.lightgbm.value, objective: str = ObjectiveName.check.value, - delta: float = 0.05, + delta: float = 0.01, ) -> None: """Initialize the MQRegressor.""" self._params = params From 39e80a6185f14f342b4b6b856631c9172e8834e9 Mon Sep 17 00:00:00 2001 From: "bbbeomjin@gmail.com" Date: Fri, 2 Aug 2024 16:57:03 +0900 Subject: [PATCH 2/2] remove the redundant function --- mqboost/objective.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/mqboost/objective.py b/mqboost/objective.py index 8095251..e7e8cc8 100644 --- a/mqboost/objective.py +++ b/mqboost/objective.py @@ -67,20 +67,15 @@ def _grad_huber(u: np.ndarray, alpha: float, delta: float) -> np.ndarray: return _r * _smaller_delta + _g * _bigger_delta -def _hess_huber(u: np.ndarray, alpha: float, delta: float) -> np.ndarray: - _h = np.ones_like(u) - return _h - - def _grad_phuber(u: np.ndarray, alpha: float, delta: float) -> np.ndarray: scale = delta**2 + u**2 - _g = -abs(_grad_rho(u, alpha)) * u / scale**(1/2) + _g = -abs(_grad_rho(u, alpha)) * u / scale ** (1 / 2) return _g def _hess_phuber(u: np.ndarray, alpha: float, delta: float) -> np.ndarray: - scale = 1 + (u / delta)**2 - _h = (1 / delta) * abs(_grad_rho(u, alpha)) / (scale**(3/2)) + scale = 1 + (u / delta) ** 2 + _h = (1 / delta) * abs(_grad_rho(u, alpha)) / (scale ** (3 / 2)) return _h @@ -127,20 +122,27 @@ def _compute_grads_hess( _y_train, _y_pred = _train_pred_reshape( y_pred=y_pred, dtrain=dtrain, len_alpha=_len_alpha ) - grads = []; hess = [] + grads = [] + hess = [] for alpha_inx in range(len(alphas)): _err_for_alpha = _y_train[alpha_inx] - _y_pred[alpha_inx] - _grad = grad_fn(u = _err_for_alpha, alpha = alphas[alpha_inx], **kwargs) - _hess = hess_fn(u = _err_for_alpha, alpha = alphas[alpha_inx], **kwargs) + _grad = grad_fn(u=_err_for_alpha, alpha=alphas[alpha_inx], **kwargs) + _hess = hess_fn(u=_err_for_alpha, alpha=alphas[alpha_inx], **kwargs) grads.append(_grad) hess.append(_hess) - + return np.concatenate(grads), np.concatenate(hess) -check_loss_grad_hess: Callable = partial(_compute_grads_hess, grad_fn=_grad_rho, hess_fn = _hess_rho) -huber_loss_grad_hess: Callable = partial(_compute_grads_hess, grad_fn=_grad_huber, hess_fn = _hess_huber) -phuber_loss_grad_hess: Callable = partial(_compute_grads_hess, grad_fn=_grad_phuber, hess_fn = _hess_phuber) +check_loss_grad_hess: Callable = partial( + _compute_grads_hess, grad_fn=_grad_rho, hess_fn=_hess_rho +) +huber_loss_grad_hess: Callable = partial( + _compute_grads_hess, grad_fn=_grad_huber, hess_fn=_hess_rho +) +phuber_loss_grad_hess: Callable = partial( + _compute_grads_hess, grad_fn=_grad_phuber, hess_fn=_hess_phuber +) def _eval_check_loss( @@ -238,8 +240,10 @@ def __init__( elif objective == ObjectiveName.check: self._fobj = partial(check_loss_grad_hess, alphas=alphas) elif objective == ObjectiveName.phuber: - self._delta = delta_validate(delta = delta) - self._fobj = partial(phuber_loss_grad_hess, alphas = alphas, delta = self._delta) + self._delta = delta_validate(delta=delta) + self._fobj = partial( + phuber_loss_grad_hess, alphas=alphas, delta=self._delta + ) self._eval_name = CHECK_LOSS if model == ModelName.lightgbm: