[python-package] Add decision_function() to LGBMClassifier (#7159)

Co-authored-by: Wota <[email protected]>
Co-authored-by: James Lamb <[email protected]>

Author: 3 people

python-package/lightgbm/sklearn.py

@@ -727,25 +727,35 @@ def _more_tags(self) -> Dict[str, Any]:
         # "check_sample_weight_equivalence" can be removed when lightgbm's
         # minimum supported scikit-learn version is at least 1.6
         # ref: https://github.com/scikit-learn/scikit-learn/pull/30137
+        xfail_checks = {
+            "check_no_attributes_set_in_init": (
+                "scikit-learn incorrectly asserts that private attributes "
+                "cannot be set in __init__: "
+                "(see https://github.com/lightgbm-org/LightGBM/issues/2628)"
+            ),
+            "check_all_zero_sample_weights_error": (
+                "Beginning in scikit-learn 1.9, by default estimators are expected to reject "
+                "sample weight arrays that are all-0. LightGBM intentionally accepts such arrays. "
+                "LightGBM supports some operations where training on an all-0-weight input could make sense, "
+                "like batch updates with training continuation or manual model creation with forced splits."
+            ),
+            "check_sample_weight_equivalence": check_sample_weight_str,
+            "check_sample_weight_equivalence_on_dense_data": check_sample_weight_str,
+            "check_sample_weight_equivalence_on_sparse_data": check_sample_weight_str,
+        }
+        # "check_decision_proba_consistency" can be removed when lightgbm's
+        # minimum supported scikit-learn version is at least 1.2
+        sklearn_major, sklearn_minor, *_ = _sklearn_version.split(".")
+        if (int(sklearn_major), int(sklearn_minor)) < (1, 2):
+            xfail_checks["check_decision_proba_consistency"] = (
+                "decision_function() returns raw margins while predict_proba() applies sigmoid in C++ "
+                "independently, causing different tie structures after rounding. "
+                "scikit-learn >= 1.2 relaxed this check to accept monotonically consistent scores."
+            )
         return {
             "allow_nan": True,
             "X_types": ["2darray", "sparse", "1dlabels"],
-            "_xfail_checks": {
-                "check_no_attributes_set_in_init": (
-                    "scikit-learn incorrectly asserts that private attributes "
-                    "cannot be set in __init__: "
-                    "(see https://github.com/lightgbm-org/LightGBM/issues/2628)"
-                ),
-                "check_all_zero_sample_weights_error": (
-                    "Beginning in scikit-learn 1.9, by default estimators are expected to reject "
-                    "sample weight arrays that are all-0. LightGBM intentionally accepts such arrays. "
-                    "LightGBM supports some operations where training on an all-0-weight input could make sense, "
-                    "like batch updates with training continuation or manual model creation with forced splits."
-                ),
-                "check_sample_weight_equivalence": check_sample_weight_str,
-                "check_sample_weight_equivalence_on_dense_data": check_sample_weight_str,
-                "check_sample_weight_equivalence_on_sparse_data": check_sample_weight_str,
-            },
+            "_xfail_checks": xfail_checks,
         }
 
     @staticmethod
@@ -1737,6 +1747,49 @@ def predict_proba(
         X_SHAP_values_shape="array-like of shape = [n_samples, n_features + 1] or shape = [n_samples, (n_features + 1) * n_classes] or list with n_classes length of such objects",
     )
 
+    def decision_function(
+        self,
+        X: _LGBM_ScikitMatrixLike,
+        *,
+        start_iteration: int = 0,
+        num_iteration: Optional[int] = None,
+        validate_features: bool = False,
+        **kwargs: Any,
+    ) -> _LGBM_PredictReturnType:
+        """Return the raw margin score for each sample.
+
+        Parameters
+        ----------
+        X : numpy array, pandas DataFrame, scipy.sparse, list of lists of int or float of shape = [n_samples, n_features]
+            Input features matrix.
+        start_iteration : int, optional (default=0)
+            Start index of the iteration to predict.
+            If <= 0, starts from the first iteration.
+        num_iteration : int or None, optional (default=None)
+            Total number of iterations used in the prediction.
+            If None, if the best iteration exists and start_iteration <= 0, the best iteration is used;
+            otherwise, all iterations from ``start_iteration`` are used (no limits).
+            If <= 0, all iterations from ``start_iteration`` are used (no limits).
+        validate_features : bool, optional (default=False)
+            If True, ensure that the features used to predict match the ones used to train.
+            Used only if data is pandas DataFrame.
+        **kwargs
+            Other parameters forwarded to ``predict()``.
+
+        Returns
+        -------
+        raw_score : array-like of shape = [n_samples] or shape = [n_samples, n_classes]
+            The predicted values.
+        """
+        return super().predict(
+            X=X,
+            raw_score=True,
+            start_iteration=start_iteration,
+            num_iteration=num_iteration,
+            validate_features=validate_features,
+            **kwargs,
+        )
+
     @property
     def classes_(self) -> np.ndarray:
         """:obj:`array` of shape = [n_classes]: The class label array."""

tests/python_package_test/test_sklearn.py

@@ -13,6 +13,7 @@
 import scipy.sparse
 from scipy.stats import spearmanr
 from sklearn.base import clone
+from sklearn.calibration import CalibratedClassifierCV
 from sklearn.datasets import load_svmlight_file, make_blobs, make_multilabel_classification
 from sklearn.ensemble import StackingClassifier, StackingRegressor
 from sklearn.metrics import accuracy_score, log_loss, mean_squared_error, r2_score
@@ -41,6 +42,7 @@
     load_digits,
     load_iris,
     load_linnerud,
+    logistic_sigmoid,
     make_ranking,
     make_synthetic_regression,
     np_assert_array_equal,
@@ -973,6 +975,67 @@ def test_predict():
     np.testing.assert_allclose(res_class_sklearn, y_train)
 
 
+def test_decision_function_and_predict_proba_consistency():
+    # binary
+    X, y = load_breast_cancer(return_X_y=True)
+    X_train, X_test, y_train, _ = train_test_split(X, y, test_size=0.2, random_state=42)
+    clf = lgb.LGBMClassifier(n_estimators=10, random_state=42, verbose=-1).fit(X_train, y_train)
+    preds_raw = clf.decision_function(X_test)
+    np.testing.assert_allclose(preds_raw, clf.predict(X_test, raw_score=True))
+    np.testing.assert_allclose(logistic_sigmoid(preds_raw), clf.predict_proba(X_test)[:, 1])
+
+    # multiclass
+    X, y = load_iris(return_X_y=True)
+    X_train, X_test, y_train, _ = train_test_split(X, y, test_size=0.2, random_state=42)
+    clf = lgb.LGBMClassifier(n_estimators=10, random_state=42, verbose=-1).fit(X_train, y_train)
+    preds_raw = clf.decision_function(X_test)
+    np.testing.assert_allclose(preds_raw, clf.predict(X_test, raw_score=True))
+    np.testing.assert_allclose(softmax(preds_raw), clf.predict_proba(X_test))
+
+
+@pytest.mark.parametrize("method", ["sigmoid", "isotonic"])
+def test_calibrated_classifier_cv(method):
+    # binary
+    X, y = load_breast_cancer(return_X_y=True)
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
+    deterministic_params = {
+        "deterministic": True,
+        "force_col_wise": True,
+        "n_jobs": 1,
+        "seed": 312,
+    }
+    clf = CalibratedClassifierCV(
+        lgb.LGBMClassifier(n_estimators=10, verbose=-1, **deterministic_params),
+        method=method,
+        cv=3,
+    )
+    clf.fit(X_train, y_train)
+    proba = clf.predict_proba(X_test)
+    assert proba.shape == (X_test.shape[0], 2)
+    np.testing.assert_array_less(proba, 1.0 + 1e-9)
+    np.testing.assert_array_less(-1e-9, proba)
+    np.testing.assert_allclose(proba.sum(axis=1), 1.0)
+    score = accuracy_score(y_test, clf.predict(X_test))
+    assert 0.8 <= score <= 1.0
+
+    # multiclass
+    X, y = load_iris(return_X_y=True)
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
+    clf = CalibratedClassifierCV(
+        lgb.LGBMClassifier(n_estimators=10, verbose=-1, **deterministic_params),
+        method=method,
+        cv=3,
+    )
+    clf.fit(X_train, y_train)
+    proba = clf.predict_proba(X_test)
+    assert proba.shape == (X_test.shape[0], 3)
+    np.testing.assert_array_less(proba, 1.0 + 1e-9)
+    np.testing.assert_array_less(-1e-9, proba)
+    np.testing.assert_allclose(proba.sum(axis=1), 1.0)
+    score = accuracy_score(y_test, clf.predict(X_test))
+    assert 0.8 <= score <= 1.0
+
+
 def test_predict_with_params_from_init():
     X, y = load_iris(return_X_y=True)
     X_train, X_test, y_train, _ = train_test_split(X, y, test_size=0.2, random_state=42)