Replace nan with denormal floating point. (#11428)

- Remove nan as the init value for tree models.
- Avoid zero division in linear models.

Author: trivialfis

src/linear/coordinate_common.h

@@ -1,20 +1,21 @@
 /**
- * Copyright 2018-2023 by XGBoost Contributors
+ * Copyright 2018-2025, XGBoost Contributors
  * \author Rory Mitchell
  */
 #pragma once
 #include <algorithm>
+#include <cmath>  // for fpclassify
+#include <limits>
 #include <string>
 #include <utility>
 #include <vector>
-#include <limits>
 
-#include "xgboost/data.h"
-#include "xgboost/parameter.h"
-#include "./param.h"
-#include "../gbm/gblinear_model.h"
 #include "../common/random.h"
 #include "../common/threading_utils.h"
+#include "../gbm/gblinear_model.h"
+#include "./param.h"
+#include "xgboost/data.h"
+#include "xgboost/parameter.h"
 
 namespace xgboost {
 namespace linear {
@@ -64,7 +65,11 @@ inline double CoordinateDelta(double sum_grad, double sum_hess, double w,
  * \return  The weight update.
  */
 inline double CoordinateDeltaBias(double sum_grad, double sum_hess) {
-  return -sum_grad / sum_hess;
+  auto b = -sum_grad / sum_hess;
+  if (std::isnan(b) || std::isinf(b)) {
+    b = 0;
+  }
+  return b;
 }
 
 /**

src/linear/updater_coordinate.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2018-2023 by XGBoost Contributors
+ * Copyright 2018-2025, XGBoost Contributors
  * \author Rory Mitchell
  */
 
@@ -49,21 +49,21 @@ class CoordinateUpdater : public LinearUpdater {
               double sum_instance_weight) override {
     auto gpair = in_gpair->Data();
     tparam_.DenormalizePenalties(sum_instance_weight);
-    const int ngroup = model->learner_model_param->num_output_group;
+    auto ngroup = model->learner_model_param->num_output_group;
     // update bias
-    for (int group_idx = 0; group_idx < ngroup; ++group_idx) {
+    for (decltype(ngroup) group_idx = 0; group_idx < ngroup; ++group_idx) {
       auto grad = GetBiasGradientParallel(group_idx, ngroup, gpair->ConstHostVector(), p_fmat,
                                           ctx_->Threads());
-      auto dbias = static_cast<float>(tparam_.learning_rate *
-                                      CoordinateDeltaBias(grad.first, grad.second));
+      auto dbias =
+          static_cast<float>(tparam_.learning_rate * CoordinateDeltaBias(grad.first, grad.second));
       model->Bias()[group_idx] += dbias;
       UpdateBiasResidualParallel(ctx_, group_idx, ngroup, dbias, &gpair->HostVector(), p_fmat);
     }
     // prepare for updating the weights
     selector_->Setup(ctx_, *model, gpair->ConstHostVector(), p_fmat, tparam_.reg_alpha_denorm,
                      tparam_.reg_lambda_denorm, cparam_.top_k);
     // update weights
-    for (int group_idx = 0; group_idx < ngroup; ++group_idx) {
+    for (decltype(ngroup) group_idx = 0; group_idx < ngroup; ++group_idx) {
       for (unsigned i = 0U; i < model->learner_model_param->num_feature; i++) {
         int fidx =
             selector_->NextFeature(ctx_, i, *model, group_idx, gpair->ConstHostVector(), p_fmat,

src/tree/io_utils.h

@@ -1,11 +1,12 @@
 /**
- * Copyright 2023-2024, XGBoost Contributors
+ * Copyright 2023-2025, XGBoost Contributors
  */
 #ifndef XGBOOST_TREE_IO_UTILS_H_
 #define XGBOOST_TREE_IO_UTILS_H_
-#include <string>          // for string
-#include <type_traits>     // for enable_if_t, is_same_v, conditional_t
-#include <vector>          // for vector
+#include <limits>       // for numeric_limits
+#include <string>       // for string
+#include <type_traits>  // for enable_if_t, is_same_v, conditional_t
+#include <vector>       // for vector
 
 #include "xgboost/json.h"  // for Json
 
@@ -59,5 +60,7 @@ inline std::string const kParent{"parents"};
 inline std::string const kLeft{"left_children"};
 inline std::string const kRight{"right_children"};
 }  // namespace tree_field
+
+constexpr float DftBadValue() { return std::numeric_limits<float>::denorm_min(); }
 }  // namespace xgboost
 #endif  // XGBOOST_TREE_IO_UTILS_H_

src/tree/multi_target_tree_model.cc

@@ -25,8 +25,8 @@ MultiTargetTree::MultiTargetTree(TreeParam const* param)
       parent_(1ul, InvalidNodeId()),
       split_index_(1ul, 0),
       default_left_(1ul, 0),
-      split_conds_(1ul, std::numeric_limits<float>::quiet_NaN()),
-      weights_(param->size_leaf_vector, std::numeric_limits<float>::quiet_NaN()) {
+      split_conds_(1ul, DftBadValue()),
+      weights_(param->size_leaf_vector, DftBadValue()) {
   CHECK_GT(param_->size_leaf_vector, 1);
 }
 
@@ -222,7 +222,7 @@ void MultiTargetTree::Expand(bst_node_t nidx, bst_feature_t split_idx, float spl
   split_index_.Resize(n);
   split_index_.HostVector()[nidx] = split_idx;
 
-  split_conds_.Resize(n, std::numeric_limits<float>::quiet_NaN());
+  split_conds_.Resize(n, DftBadValue());
   split_conds_.HostVector()[nidx] = split_cond;
 
   default_left_.Resize(n);

src/tree/tree_model.cc

@@ -891,24 +891,23 @@ void RegTree::ExpandNode(bst_node_t nidx, bst_feature_t split_index, float split
   this->param_.num_nodes = this->p_mt_tree_->Size();
 }
 
-void RegTree::ExpandCategorical(bst_node_t nid, bst_feature_t split_index,
-                                common::Span<const uint32_t> split_cat, bool default_left,
-                                bst_float base_weight, bst_float left_leaf_weight,
-                                bst_float right_leaf_weight, bst_float loss_change, float sum_hess,
-                                float left_sum, float right_sum) {
+void RegTree::ExpandCategorical(bst_node_t nidx, bst_feature_t split_index,
+                                common::Span<common::KCatBitField::value_type> split_cat,
+                                bool default_left, bst_float base_weight,
+                                bst_float left_leaf_weight, bst_float right_leaf_weight,
+                                bst_float loss_change, float sum_hess, float left_sum,
+                                float right_sum) {
   CHECK(!IsMultiTarget());
-  this->ExpandNode(nid, split_index, std::numeric_limits<float>::quiet_NaN(),
-                   default_left, base_weight,
-                   left_leaf_weight, right_leaf_weight, loss_change, sum_hess,
-                   left_sum, right_sum);
+  this->ExpandNode(nidx, split_index, DftBadValue(), default_left, base_weight, left_leaf_weight,
+                   right_leaf_weight, loss_change, sum_hess, left_sum, right_sum);
 
   size_t orig_size = split_categories_.size();
   this->split_categories_.resize(orig_size + split_cat.size());
   std::copy(split_cat.data(), split_cat.data() + split_cat.size(),
             split_categories_.begin() + orig_size);
-  this->split_types_.at(nid) = FeatureType::kCategorical;
-  this->split_categories_segments_.at(nid).beg = orig_size;
-  this->split_categories_segments_.at(nid).size = split_cat.size();
+  this->split_types_.at(nidx) = FeatureType::kCategorical;
+  this->split_categories_segments_.at(nidx).beg = orig_size;
+  this->split_categories_segments_.at(nidx).size = split_cat.size();
 }
 
 void RegTree::Load(dmlc::Stream* fi) {

tests/cpp/tree/test_tree_model.cc

@@ -1,10 +1,11 @@
 /**
- * Copyright 2018-2024, XGBoost Contributors
+ * Copyright 2018-2025, XGBoost Contributors
  */
 #include <gtest/gtest.h>
 
 #include "../../../src/common/bitfield.h"
 #include "../../../src/common/categorical.h"
+#include "../../../src/tree/io_utils.h"  // for DftBadValue
 #include "../filesystem.h"
 #include "../helpers.h"
 #include "xgboost/tree_model.h"
@@ -166,7 +167,7 @@ TEST(Tree, ExpandCategoricalFeature) {
     ASSERT_EQ(tree.GetSplitTypes()[1], FeatureType::kNumerical);
     ASSERT_EQ(tree.GetSplitTypes()[2], FeatureType::kNumerical);
     ASSERT_EQ(tree.GetSplitCategories().size(), 0ul);
-    ASSERT_TRUE(std::isnan(tree[0].SplitCond()));
+    ASSERT_EQ(tree[0].SplitCond(), DftBadValue());
   }
   {
     RegTree tree;

tests/cpp/tree/test_tree_stat.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020-2024, XGBoost Contributors
+ * Copyright 2020-2025, XGBoost Contributors
  */
 #include <gtest/gtest.h>
 #include <xgboost/context.h>       // for Context
@@ -9,7 +9,8 @@
 
 #include <memory>  // for unique_ptr
 
-#include "../../../src/tree/param.h"  // for TrainParam
+#include "../../../src/tree/io_utils.h"  // for DftBadValue
+#include "../../../src/tree/param.h"     // for TrainParam
 #include "../helpers.h"
 
 namespace xgboost {
@@ -127,8 +128,8 @@ class TestSplitWithEta : public ::testing::Test {
           for (std::size_t i = 0; i < leaf_0.Size(); ++i) {
             CHECK_EQ(leaf_0(i) * eta_ratio, leaf_1(i));
           }
-          CHECK(std::isnan(p_tree0->SplitCond(nidx)));
-          CHECK(std::isnan(p_tree1->SplitCond(nidx)));
+          CHECK_EQ(DftBadValue(), p_tree0->SplitCond(nidx));
+          CHECK_EQ(DftBadValue(), p_tree1->SplitCond(nidx));
         } else {
           // NON-mt tree reuses split cond for leaf value.
           auto leaf_0 = p_tree0->SplitCond(nidx);

tests/python/test_linear.py

@@ -1,3 +1,5 @@
+from typing import Dict
+
 from hypothesis import given, note, settings, strategies
 
 import xgboost as xgb
@@ -20,8 +22,8 @@
 })
 
 
-def train_result(param, dmat, num_rounds):
-    result = {}
+def train_result(param: dict, dmat: xgb.DMatrix, num_rounds: int) -> Dict[str, Dict]:
+    result: Dict[str, Dict] = {}
     xgb.train(
         param,
         dmat,