Source code for neural_trees.distilled_figs

"""
FIGS fitted to a teacher's class probabilities instead of the labels.

FIGS (Tan et al., PNAS 2025; the `imodels` package) grows a small sum of
axis-aligned trees, one split at a time, under a total split budget. Fitted
on the labels it overfits as the budget grows. Fitted by regression on a
strong teacher's probabilities it keeps improving instead. Distilling into
FIGS is not new (FIGS-BD, arXiv 2503.06730; `imodels.DistilledRegressor`);
this class is the recipe measured in the equal-size audit of the
neural-trees paper.

Measured there on 9 OpenML tables x 3 seeds, 20 000 training rows each,
tuned XGBoost teacher, scored on held-out test rows that no choice was made
on (protocol and hypotheses fixed in advance):

    splits   FIGS on labels      DistilledFIGS
             acc gap   AUC       acc gap   AUC
    128      -2.86     86.44     -2.28     87.41
    256      -3.06     85.79     -1.93     87.80
    512      -3.58     84.31     -1.84     87.95

(acc gap = accuracy minus the teacher's, in points; teacher AUC 89.88.)
At 512 splits it is ahead of FIGS on labels at its best size (128) by
+1.02 accuracy and +1.52 AUC, on every one of the 9 tables, and about level
with an EBM without interactions (+0.68 accuracy, 95% interval +0.16 to
+1.25). A full EBM with interactions stays ahead by about one point.
Synthetic rows (MUNGE) gave the same result at 13 times the fit time, so
`synthetic` defaults to 0.

Needs `imodels` (``pip install neural-trees[figs]``).
"""
from __future__ import annotations

from typing import Optional

import numpy as np
from sklearn.base import BaseEstimator, ClassifierMixin, clone
from sklearn.utils.multiclass import check_classification_targets
from sklearn.utils.validation import check_array, check_is_fitted, check_X_y

__all__ = ["DistilledFIGS"]


[docs] class DistilledFIGS(ClassifierMixin, BaseEstimator): """ A FIGS model trained on what a strong model learned: `fit` trains the `teacher` on (X, y), then a `FIGSRegressor` with `max_rules` splits on the teacher's probability of each class (one regressor for two classes, one per class otherwise). Prediction uses only the FIGS model(s); the teacher is kept to measure agreement. Parameters ---------- teacher : classifier with predict_proba, default=None Cloned and fitted on (X, y). None uses scikit-learn's `HistGradientBoostingClassifier()`. The measurements in the module docstring used a tuned XGBoost teacher. max_rules : int, default=256 Split budget of each FIGS regressor. 256 to 512 is where the gain over FIGS on labels was measured; with several classes the total size is ``n_classes * max_rules``. synthetic : int, default=0 MUNGE passes over the rows (see `neural_trees.distillation.munge`), labelled by the teacher for the student. Did not help in the measurements; kept for small tables. random_state : int, optional Seeds FIGS and the synthetic rows. Attributes ---------- teacher_ : the fitted teacher figs_ : list of fitted `imodels.FIGSRegressor`, one per modelled class classes_, n_features_in_ complexity_ : int, total number of splits over `figs_` """ def __init__(self, teacher=None, max_rules: int = 256, synthetic: int = 0, random_state: Optional[int] = None): self.teacher = teacher self.max_rules = max_rules self.synthetic = synthetic self.random_state = random_state def fit(self, X, y, sample_weight=None) -> DistilledFIGS: try: from imodels import FIGSRegressor except ImportError as e: # pragma: no cover - depends on the environment raise ImportError("DistilledFIGS needs imodels: pip install neural-trees[figs]") from e from sklearn.ensemble import HistGradientBoostingClassifier from neural_trees.distillation import munge if not isinstance(self.max_rules, (int, np.integer)) or isinstance(self.max_rules, bool) or self.max_rules < 1: raise ValueError(f"max_rules must be a positive int, got {self.max_rules!r}") if not isinstance(self.synthetic, (int, np.integer)) or isinstance(self.synthetic, bool) or self.synthetic < 0: raise ValueError(f"synthetic must be a non-negative int, got {self.synthetic!r}") X, y = check_X_y(X, y, dtype=np.float64) check_classification_targets(y) teacher = clone(self.teacher) if self.teacher is not None else HistGradientBoostingClassifier() if sample_weight is None: teacher.fit(X, y) else: teacher.fit(X, y, sample_weight=sample_weight) self.classes_ = np.asarray(teacher.classes_) if len(self.classes_) < 2: raise ValueError("DistilledFIGS needs at least two classes in y") Xt = X if self.synthetic: Xt = np.vstack([X, munge(X, passes=int(self.synthetic), random_state=self.random_state)]) P = np.asarray(teacher.predict_proba(Xt), dtype=np.float64) targets = [P[:, 1]] if len(self.classes_) == 2 else [P[:, k] for k in range(len(self.classes_))] self.figs_ = [FIGSRegressor(max_rules=int(self.max_rules), random_state=self.random_state).fit(Xt, t) for t in targets] self.teacher_ = teacher self.n_features_in_ = X.shape[1] self.complexity_ = int(sum(m.complexity_ for m in self.figs_)) return self def predict_proba(self, X) -> np.ndarray: check_is_fitted(self) X = check_array(X, dtype=np.float64) if X.shape[1] != self.n_features_in_: raise ValueError(f"X has {X.shape[1]} features, the model was fitted on {self.n_features_in_}") S = np.column_stack([np.asarray(m.predict(X), dtype=np.float64) for m in self.figs_]) S = np.clip(S, 0.0, 1.0) if len(self.classes_) == 2: return np.column_stack([1.0 - S[:, 0], S[:, 0]]) tot = S.sum(1, keepdims=True) return np.where(tot > 0, S / np.where(tot > 0, tot, 1.0), 1.0 / len(self.classes_)) def predict(self, X) -> np.ndarray: return self.classes_[np.argmax(self.predict_proba(X), axis=1)]
[docs] def fidelity(self, X) -> float: """Share of rows of `X` where this model predicts the teacher's class.""" check_is_fitted(self) X = check_array(X, dtype=np.float64) return float(np.mean(self.predict(X) == np.asarray(self.teacher_.predict(X))))