Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Auc mu weights #3349

Merged
merged 4 commits into from
Sep 20, 2020
Merged

Auc mu weights #3349

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e893f39
Update auc_mu metric to use data weights if provided
btrotta Sep 3, 2020
0d69791
Calculate class sizes and total weights in Init so we only do it once
btrotta Sep 3, 2020
def9d27
Fix lint error
btrotta Sep 3, 2020
82a298f
Empty commit to trigger CI jobs
btrotta Sep 11, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 84 additions & 30 deletions src/metric/multiclass_metric.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,30 +199,50 @@ class AucMuMetric : public Metric {
num_data_ = num_data;
label_ = metadata.label();

// get weights
weights_ = metadata.weights();
if (weights_ == nullptr) {
sum_weights_ = static_cast<double>(num_data_);
} else {
sum_weights_ = 0.0f;
for (data_size_t i = 0; i < num_data_; ++i) {
sum_weights_ += weights_[i];
}
}

// sort the data indices by true class
sorted_data_idx_ = std::vector<data_size_t>(num_data_, 0);
for (data_size_t i = 0; i < num_data_; ++i) {
sorted_data_idx_[i] = i;
}
Common::ParallelSort(sorted_data_idx_.begin(), sorted_data_idx_.end(),
[this](data_size_t a, data_size_t b) { return label_[a] < label_[b]; });
}

std::vector<double> Eval(const double* score, const ObjectiveFunction*) const override {
// the notation follows that used in the paper introducing the auc-mu metric:
// http://proceedings.mlr.press/v97/kleiman19a/kleiman19a.pdf

// get size of each class
auto class_sizes = std::vector<data_size_t>(num_class_, 0);
class_sizes_ = std::vector<data_size_t>(num_class_, 0);
for (data_size_t i = 0; i < num_data_; ++i) {
data_size_t curr_label = static_cast<data_size_t>(label_[i]);
++class_sizes[curr_label];
++class_sizes_[curr_label];
}

// get total weight of data in each class
class_data_weights_ = std::vector<double>(num_class_, 0);
if (weights_ != nullptr) {
for (data_size_t i = 0; i < num_data_; ++i) {
data_size_t curr_label = static_cast<data_size_t>(label_[i]);
class_data_weights_[curr_label] += weights_[i];
}
}
}

std::vector<double> Eval(const double* score, const ObjectiveFunction*) const override {
// the notation follows that used in the paper introducing the auc-mu metric:
// http://proceedings.mlr.press/v97/kleiman19a/kleiman19a.pdf

auto S = std::vector<std::vector<double>>(num_class_, std::vector<double>(num_class_, 0));
int i_start = 0;
for (int i = 0; i < num_class_; ++i) {
int j_start = i_start + class_sizes[i];
int j_start = i_start + class_sizes_[i];
for (int j = i + 1; j < num_class_; ++j) {
std::vector<double> curr_v;
for (int k = 0; k < num_class_; ++k) {
Expand All @@ -231,9 +251,9 @@ class AucMuMetric : public Metric {
double t1 = curr_v[i] - curr_v[j];
// extract the data indices belonging to class i or j
std::vector<data_size_t> class_i_j_indices;
class_i_j_indices.assign(sorted_data_idx_.begin() + i_start, sorted_data_idx_.begin() + i_start + class_sizes[i]);
class_i_j_indices.assign(sorted_data_idx_.begin() + i_start, sorted_data_idx_.begin() + i_start + class_sizes_[i]);
class_i_j_indices.insert(class_i_j_indices.end(),
sorted_data_idx_.begin() + j_start, sorted_data_idx_.begin() + j_start + class_sizes[j]);
sorted_data_idx_.begin() + j_start, sorted_data_idx_.begin() + j_start + class_sizes_[j]);
// sort according to distance from separating hyperplane
std::vector<std::pair<data_size_t, double>> dist;
for (data_size_t k = 0; static_cast<size_t>(k) < class_i_j_indices.size(); ++k) {
Expand All @@ -259,38 +279,64 @@ class AucMuMetric : public Metric {
double num_j = 0;
double last_j_dist = 0;
double num_current_j = 0;
for (size_t k = 0; k < dist.size(); ++k) {
data_size_t a = dist[k].first;
double curr_dist = dist[k].second;
if (label_[a] == i) {
if (std::fabs(curr_dist - last_j_dist) < kEpsilon) {
S[i][j] += num_j - 0.5 * num_current_j; // members of class j with same distance as a contribute 0.5
if (weights_ == nullptr) {
for (size_t k = 0; k < dist.size(); ++k) {
data_size_t a = dist[k].first;
double curr_dist = dist[k].second;
if (label_[a] == i) {
if (std::fabs(curr_dist - last_j_dist) < kEpsilon) {
S[i][j] += num_j - 0.5 * num_current_j; // members of class j with same distance as a contribute 0.5
} else {
S[i][j] += num_j;
}
} else {
S[i][j] += num_j;
++num_j;
if (std::fabs(curr_dist - last_j_dist) < kEpsilon) {
++num_current_j;
} else {
last_j_dist = dist[k].second;
num_current_j = 1;
}
}
} else {
++num_j;
if (std::fabs(curr_dist - last_j_dist) < kEpsilon) {
++num_current_j;
}
} else {
for (size_t k = 0; k < dist.size(); ++k) {
data_size_t a = dist[k].first;
double curr_dist = dist[k].second;
double curr_weight = weights_[a];
if (label_[a] == i) {
if (std::fabs(curr_dist - last_j_dist) < kEpsilon) {
S[i][j] += curr_weight * (num_j - 0.5 * num_current_j); // members of class j with same distance as a contribute 0.5
} else {
S[i][j] += curr_weight * num_j;
}
} else {
last_j_dist = dist[k].second;
num_current_j = 1;
num_j += curr_weight;
if (std::fabs(curr_dist - last_j_dist) < kEpsilon) {
num_current_j += curr_weight;
} else {
last_j_dist = dist[k].second;
num_current_j = curr_weight;
}
}
}
}
j_start += class_sizes[j];
j_start += class_sizes_[j];
}
i_start += class_sizes[i];
i_start += class_sizes_[i];
}

double ans = 0;
for (int i = 0; i < num_class_; ++i) {
for (int j = i + 1; j < num_class_; ++j) {
ans += (S[i][j] / class_sizes[i]) / class_sizes[j];
if (weights_ == nullptr) {
ans += (S[i][j] / class_sizes_[i]) / class_sizes_[j];
} else {
ans += (S[i][j] / class_data_weights_[i]) / class_data_weights_[j];
}
}
}
ans = (2 * ans / num_class_) / (num_class_ - 1);
return std::vector<double>(1, ans);
ans = (2.0 * ans / num_class_) / (num_class_ - 1);
return std::vector<double>(1.0, ans);
}

private:
Expand All @@ -302,8 +348,16 @@ class AucMuMetric : public Metric {
std::vector<std::string> name_;
/*! \brief Number of classes*/
int num_class_;
/*! \brief class_weights*/
/*! \brief Class auc-mu weights*/
std::vector<std::vector<double>> class_weights_;
/*! \brief Data weights */
const label_t* weights_;
/*! \brief Sum of data weights */
double sum_weights_;
/*! \brief Sum of data weights in each class*/
std::vector<double> class_data_weights_;
/*! \brief Number of data in each class*/
std::vector<data_size_t> class_sizes_;
/*! \brief config parameters*/
Config config_;
/*! \brief index to data, sorted by true class*/
Expand Down
19 changes: 18 additions & 1 deletion tests/python_package_test/test_engine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -526,6 +526,23 @@ def test_auc_mu(self):
results_auc_mu = {}
lgb.train(params, lgb_X, num_boost_round=10, valid_sets=[lgb_X], evals_result=results_auc_mu)
self.assertAlmostEqual(results_auc_mu['training']['auc_mu'][-1], 0.5)
# test that weighted data gives different auc_mu
lgb_X = lgb.Dataset(X, label=y)
lgb_X_weighted = lgb.Dataset(X, label=y, weight=np.abs(np.random.normal(size=y.shape)))
results_unweighted = {}
results_weighted = {}
params = dict(params, num_classes=10, num_leaves=5)
lgb.train(params, lgb_X, num_boost_round=10, valid_sets=[lgb_X], evals_result=results_unweighted)
lgb.train(params, lgb_X_weighted, num_boost_round=10, valid_sets=[lgb_X_weighted],
evals_result=results_weighted)
self.assertLess(results_weighted['training']['auc_mu'][-1], 1)
self.assertNotEqual(results_unweighted['training']['auc_mu'][-1], results_weighted['training']['auc_mu'][-1])
# test that equal data weights give same auc_mu as unweighted data
lgb_X_weighted = lgb.Dataset(X, label=y, weight=np.ones(y.shape) * 0.5)
lgb.train(params, lgb_X_weighted, num_boost_round=10, valid_sets=[lgb_X_weighted],
evals_result=results_weighted)
self.assertAlmostEqual(results_unweighted['training']['auc_mu'][-1], results_weighted['training']['auc_mu'][-1],
places=5)
# should give 1 when accuracy = 1
X = X[:10, :]
y = y[:10]
Expand All @@ -538,7 +555,7 @@ def test_auc_mu(self):
results = {}
lgb.train(params, lgb_X, num_boost_round=100, valid_sets=[lgb_X], evals_result=results)
self.assertAlmostEqual(results['training']['auc_mu'][-1], 1)
# test loading weights
# test loading class weights
Xy = np.loadtxt(os.path.join(os.path.dirname(os.path.realpath(__file__)),
'../../examples/multiclass_classification/multiclass.train'))
y = Xy[:, 0]
Expand Down