Skip to content

Commit

Permalink
Monotone constraints for gpu_hist (dmlc#2904)
Browse files Browse the repository at this point in the history
  • Loading branch information
@RAMitchell
RAMitchell authored Nov 29, 2017
1 parent 5867c1b commit c51adb4
Show file tree
Hide file tree
Showing 8 changed files with 172 additions and 60 deletions.
2 changes: 2 additions & 0 deletions doc/gpu/index.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ Specify the 'tree_method' parameter as one of the following algorithms.
+--------------------+------------+-----------+
| grow_policy | |cross| | |tick| |
+--------------------+------------+-----------+
| monotone_constraints | |cross| | |tick| |
+--------------------+------------+-----------+
```

Expand Down
32 changes: 21 additions & 11 deletions src/tree/param.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,10 @@ struct XGBOOST_ALIGNAS(16) GradStats {
static const int kSimpleStats = 1;
/*! \brief constructor, the object must be cleared during construction */
explicit GradStats(const TrainParam& param) { this->Clear(); }

template <typename gpair_t>
XGBOOST_DEVICE explicit GradStats(const gpair_t &sum)
: sum_grad(sum.GetGrad()), sum_hess(sum.GetHess()) {}
/*! \brief clear the statistics */
inline void Clear() { sum_grad = sum_hess = 0.0f; }
/*! \brief check if necessary information is ready */
Expand All @@ -332,11 +336,13 @@ struct XGBOOST_ALIGNAS(16) GradStats {
this->Add(b.GetGrad(), b.GetHess());
}
/*! \brief calculate leaf weight */
inline double CalcWeight(const TrainParam& param) const {
template <typename param_t>
inline double CalcWeight(const param_t& param) const {
return xgboost::tree::CalcWeight(param, sum_grad, sum_hess);
}
/*! \brief calculate gain of the solution */
inline double CalcGain(const TrainParam& param) const {
template <typename param_t>
inline double CalcGain(const param_t& param) const {
return xgboost::tree::CalcGain(param, sum_grad, sum_hess);
}
/*! \brief add statistics to the data */
Expand Down Expand Up @@ -367,7 +373,9 @@ struct XGBOOST_ALIGNAS(16) GradStats {
};

struct NoConstraint {
inline static void Init(TrainParam *param, unsigned num_feature) {}
inline static void Init(TrainParam *param, unsigned num_feature) {
param->monotone_constraints.resize(num_feature, 0);
}
inline double CalcSplitGain(const TrainParam &param, bst_uint split_index,
GradStats left, GradStats right) const {
return left.CalcGain(param) + right.CalcGain(param);
Expand All @@ -386,13 +394,14 @@ struct NoConstraint {
struct ValueConstraint {
double lower_bound;
double upper_bound;
ValueConstraint()
XGBOOST_DEVICE ValueConstraint()
: lower_bound(-std::numeric_limits<double>::max()),
upper_bound(std::numeric_limits<double>::max()) {}
inline static void Init(TrainParam *param, unsigned num_feature) {
param->monotone_constraints.resize(num_feature, 1);
param->monotone_constraints.resize(num_feature, 0);
}
inline double CalcWeight(const TrainParam &param, GradStats stats) const {
template <typename param_t>
XGBOOST_DEVICE inline double CalcWeight(const param_t &param, GradStats stats) const {
double w = stats.CalcWeight(param);
if (w < lower_bound) {
return lower_bound;
Expand All @@ -403,22 +412,23 @@ struct ValueConstraint {
return w;
}

inline double CalcGain(const TrainParam &param, GradStats stats) const {
template <typename param_t>
XGBOOST_DEVICE inline double CalcGain(const param_t &param, GradStats stats) const {
return CalcGainGivenWeight(param, stats.sum_grad, stats.sum_hess,
CalcWeight(param, stats));
}

inline double CalcSplitGain(const TrainParam &param, bst_uint split_index,
template <typename param_t>
XGBOOST_DEVICE inline double CalcSplitGain(const param_t &param, int constraint,
GradStats left, GradStats right) const {
double wleft = CalcWeight(param, left);
double wright = CalcWeight(param, right);
int c = param.monotone_constraints[split_index];
double gain =
CalcGainGivenWeight(param, left.sum_grad, left.sum_hess, wleft) +
CalcGainGivenWeight(param, right.sum_grad, right.sum_hess, wright);
if (c == 0) {
if (constraint == 0) {
return gain;
} else if (c > 0) {
} else if (constraint > 0) {
return wleft < wright ? gain : 0.0;
} else {
return wleft > wright ? gain : 0.0;
Expand Down
57 changes: 41 additions & 16 deletions src/tree/updater_colmaker.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -319,7 +319,9 @@ class ColMaker: public TreeUpdater {
if (c.sum_hess >= param.min_child_weight &&
e.stats.sum_hess >= param.min_child_weight) {
bst_float loss_chg = static_cast<bst_float>(
constraints_[nid].CalcSplitGain(param, fid, e.stats, c) - snode[nid].root_gain);
constraints_[nid].CalcSplitGain(
param, param.monotone_constraints[fid], e.stats, c) -
snode[nid].root_gain);
e.best.Update(loss_chg, fid, fsplit, false);
}
}
Expand All @@ -329,7 +331,9 @@ class ColMaker: public TreeUpdater {
if (c.sum_hess >= param.min_child_weight &&
tmp.sum_hess >= param.min_child_weight) {
bst_float loss_chg = static_cast<bst_float>(
constraints_[nid].CalcSplitGain(param, fid, tmp, c) - snode[nid].root_gain);
constraints_[nid].CalcSplitGain(
param, param.monotone_constraints[fid], tmp, c) -
snode[nid].root_gain);
e.best.Update(loss_chg, fid, fsplit, true);
}
}
Expand All @@ -341,7 +345,9 @@ class ColMaker: public TreeUpdater {
if (c.sum_hess >= param.min_child_weight &&
tmp.sum_hess >= param.min_child_weight) {
bst_float loss_chg = static_cast<bst_float>(
constraints_[nid].CalcSplitGain(param, fid, tmp, c) - snode[nid].root_gain);
constraints_[nid].CalcSplitGain(
param, param.monotone_constraints[fid], tmp, c) -
snode[nid].root_gain);
e.best.Update(loss_chg, fid, e.last_fvalue + rt_eps, true);
}
}
Expand Down Expand Up @@ -372,9 +378,11 @@ class ColMaker: public TreeUpdater {
if (c.sum_hess >= param.min_child_weight &&
e.stats.sum_hess >= param.min_child_weight) {
bst_float loss_chg = static_cast<bst_float>(
constraints_[nid].CalcSplitGain(param, fid, e.stats, c) -
constraints_[nid].CalcSplitGain(
param, param.monotone_constraints[fid], e.stats, c) -
snode[nid].root_gain);
e.best.Update(loss_chg, fid, (fvalue + e.first_fvalue) * 0.5f, false);
e.best.Update(loss_chg, fid, (fvalue + e.first_fvalue) * 0.5f,
false);
}
}
if (need_backward) {
Expand All @@ -383,7 +391,8 @@ class ColMaker: public TreeUpdater {
if (c.sum_hess >= param.min_child_weight &&
cright.sum_hess >= param.min_child_weight) {
bst_float loss_chg = static_cast<bst_float>(
constraints_[nid].CalcSplitGain(param, fid, c, cright) -
constraints_[nid].CalcSplitGain(
param, param.monotone_constraints[fid], c, cright) -
snode[nid].root_gain);
e.best.Update(loss_chg, fid, (fvalue + e.first_fvalue) * 0.5f, true);
}
Expand Down Expand Up @@ -414,12 +423,17 @@ class ColMaker: public TreeUpdater {
bst_float loss_chg;
if (d_step == -1) {
loss_chg = static_cast<bst_float>(
constraints_[nid].CalcSplitGain(param, fid, c, e.stats) - snode[nid].root_gain);
constraints_[nid].CalcSplitGain(
param, param.monotone_constraints[fid], c, e.stats) -
snode[nid].root_gain);
} else {
loss_chg = static_cast<bst_float>(
constraints_[nid].CalcSplitGain(param, fid, e.stats, c) - snode[nid].root_gain);
constraints_[nid].CalcSplitGain(
param, param.monotone_constraints[fid], e.stats, c) -
snode[nid].root_gain);
}
e.best.Update(loss_chg, fid, (fvalue + e.last_fvalue) * 0.5f, d_step == -1);
e.best.Update(loss_chg, fid, (fvalue + e.last_fvalue) * 0.5f,
d_step == -1);
}
}
// update the statistics
Expand Down Expand Up @@ -492,10 +506,14 @@ class ColMaker: public TreeUpdater {
bst_float loss_chg;
if (d_step == -1) {
loss_chg = static_cast<bst_float>(
constraints_[nid].CalcSplitGain(param, fid, c, e.stats) - snode[nid].root_gain);
constraints_[nid].CalcSplitGain(
param, param.monotone_constraints[fid], c, e.stats) -
snode[nid].root_gain);
} else {
loss_chg = static_cast<bst_float>(
constraints_[nid].CalcSplitGain(param, fid, e.stats, c) - snode[nid].root_gain);
constraints_[nid].CalcSplitGain(
param, param.monotone_constraints[fid], e.stats, c) -
snode[nid].root_gain);
}
const bst_float gap = std::abs(e.last_fvalue) + rt_eps;
const bst_float delta = d_step == +1 ? gap: -gap;
Expand Down Expand Up @@ -545,11 +563,13 @@ class ColMaker: public TreeUpdater {
bst_float loss_chg;
if (d_step == -1) {
loss_chg = static_cast<bst_float>(
constraints_[nid].CalcSplitGain(param, fid, c, e.stats) -
constraints_[nid].CalcSplitGain(
param, param.monotone_constraints[fid], c, e.stats) -
snode[nid].root_gain);
} else {
loss_chg = static_cast<bst_float>(
constraints_[nid].CalcSplitGain(param, fid, e.stats, c) -
constraints_[nid].CalcSplitGain(
param, param.monotone_constraints[fid], e.stats, c) -
snode[nid].root_gain);
}
e.best.Update(loss_chg, fid, (fvalue + e.last_fvalue) * 0.5f, d_step == -1);
Expand All @@ -565,14 +585,19 @@ class ColMaker: public TreeUpdater {
const int nid = qexpand[i];
ThreadEntry &e = temp[nid];
c.SetSubstract(snode[nid].stats, e.stats);
if (e.stats.sum_hess >= param.min_child_weight && c.sum_hess >= param.min_child_weight) {
if (e.stats.sum_hess >= param.min_child_weight &&
c.sum_hess >= param.min_child_weight) {
bst_float loss_chg;
if (d_step == -1) {
loss_chg = static_cast<bst_float>(
constraints_[nid].CalcSplitGain(param, fid, c, e.stats) - snode[nid].root_gain);
constraints_[nid].CalcSplitGain(
param, param.monotone_constraints[fid], c, e.stats) -
snode[nid].root_gain);
} else {
loss_chg = static_cast<bst_float>(
constraints_[nid].CalcSplitGain(param, fid, e.stats, c) - snode[nid].root_gain);
constraints_[nid].CalcSplitGain(
param, param.monotone_constraints[fid], e.stats, c) -
snode[nid].root_gain);
}
const bst_float gap = std::abs(e.last_fvalue) + rt_eps;
const bst_float delta = d_step == +1 ? gap: -gap;
Expand Down
4 changes: 2 additions & 2 deletions src/tree/updater_gpu.cu
View file
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,7 @@ DEV_INLINE void argMaxWithAtomics(
ExactSplitCandidate s;
bst_gpair missing = parentSum - colSum;
s.score = loss_chg_missing(gradScans[id], missing, parentSum, parentGain,
param, tmp);
param, 0, ValueConstraint(), tmp);
s.index = id;
atomicArgMax(nodeSplits + uid, s);
} // end if nodeId != UNUSED_NODE
Expand Down Expand Up @@ -580,7 +580,7 @@ class GPUMaker : public TreeUpdater {
// get the default direction for the current node
bst_gpair missing = n.sum_gradients - gradSum;
loss_chg_missing(gradScan, missing, n.sum_gradients, n.root_gain,
gpu_param, missingLeft);
gpu_param, 0, ValueConstraint(), missingLeft);
// get the score/weight/id/gradSum for left and right child nodes
bst_gpair lGradSum = missingLeft ? gradScan + missing : gradScan;
bst_gpair rGradSum = n.sum_gradients - lGradSum;
Expand Down
31 changes: 16 additions & 15 deletions src/tree/updater_gpu_common.cuh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@

#else
__device__ __forceinline__ double atomicAdd(double* address, double val) {
unsigned long long int* address_as_ull = (unsigned long long int*)address; // NOLINT
unsigned long long int* address_as_ull =
(unsigned long long int*)address; // NOLINT
unsigned long long int old = *address_as_ull, assumed; // NOLINT

do {
Expand Down Expand Up @@ -240,23 +241,23 @@ __device__ inline float device_calc_loss_chg(const GPUTrainingParam& param,
}

template <typename gpair_t>
__device__ float inline loss_chg_missing(const gpair_t& scan,
const gpair_t& missing,
const gpair_t& parent_sum,
const float& parent_gain,
const GPUTrainingParam& param,
bool& missing_left_out) { // NOLINT
float missing_left_loss =
device_calc_loss_chg(param, scan + missing, parent_sum, parent_gain);
float missing_right_loss =
device_calc_loss_chg(param, scan, parent_sum, parent_gain);

if (missing_left_loss >= missing_right_loss) {
__device__ float inline loss_chg_missing(
const gpair_t& scan, const gpair_t& missing, const gpair_t& parent_sum,
const float& parent_gain, const GPUTrainingParam& param, int constraint,
const ValueConstraint& value_constraint,
bool& missing_left_out) { // NOLINT
float missing_left_gain = value_constraint.CalcSplitGain(
param, constraint, GradStats(scan + missing),
GradStats(parent_sum - (scan + missing)));
float missing_right_gain = value_constraint.CalcSplitGain(
param, constraint, GradStats(scan), GradStats(parent_sum - scan));

if (missing_left_gain >= missing_right_gain) {
missing_left_out = true;
return missing_left_loss;
return missing_left_gain - parent_gain;
} else {
missing_left_out = false;
return missing_right_loss;
return missing_right_gain - parent_gain;
}
}

Expand Down
Loading

0 comments on commit c51adb4

Please sign in to comment.