[SC-17187] ZD-717: GINITable fails on multiclass models (follow-up to PR #535)

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "validmind"
-version = "2.13.6"
+version = "2.13.7"
 description = "ValidMind Library"
 readme = "README.pypi.md"
 requires-python = ">=3.9,<3.15"

r/validmind/DESCRIPTION

@@ -1,7 +1,7 @@
 Package: validmind
 Type: Package
 Title: Interface to the 'ValidMind' Platform
-Version: 2.13.6
+Version: 2.13.7
 Authors@R: c(person("Andres", "Rodriguez", role = c("aut", "cre","cph"),
                       email = "andres@validmind.ai"))
 Maintainer: Andres Rodriguez <andres@validmind.ai>

tests/unit_tests/model_validation/statsmodels/test_GINITable.py

@@ -1,9 +1,11 @@
 import unittest
 import pandas as pd
 import numpy as np
+from sklearn.datasets import make_classification
 from sklearn.linear_model import LogisticRegression
 import validmind as vm
 from validmind import RawData
+from validmind.errors import SkipTestError
 from validmind.tests.model_validation.statsmodels.GINITable import GINITable
 
 
@@ -180,3 +182,61 @@ def test_metric_ranges(self):
         self.assertAlmostEqual(
             results["GINI"].iloc[0], 2 * results["AUC"].iloc[0] - 1, places=5
         )
+
+
+class TestGINITableMulticlass(unittest.TestCase):
+    def setUp(self):
+        # 3-class target with real signal so per-class AUCs beat random.
+        X, y = make_classification(
+            n_samples=300,
+            n_features=5,
+            n_informative=4,
+            n_redundant=0,
+            n_classes=3,
+            n_clusters_per_class=1,
+            random_state=42,
+        )
+        df = pd.DataFrame(X, columns=[f"f{i}" for i in range(5)])
+        df["target"] = y
+
+        model = LogisticRegression(max_iter=1000)
+        model.fit(X, y)
+
+        self.vm_dataset = vm.init_dataset(
+            input_id="mc_dataset", dataset=df, target_column="target", __log=False
+        )
+        self.vm_model = vm.init_model(
+            input_id="mc_model", model=model, __log=False
+        )
+        self.vm_dataset.assign_predictions(self.vm_model)
+
+    def test_multiclass_one_vs_rest_rows(self):
+        results, raw_data = GINITable(self.vm_dataset, self.vm_model)
+
+        self.assertIsInstance(results, pd.DataFrame)
+        self.assertIsInstance(raw_data, RawData)
+
+        # One row per class + a micro-average row.
+        self.assertEqual(set(results["Class"]), {"0", "1", "2", "micro"})
+        self.assertEqual(len(results), 4)
+        for col in ("Class", "AUC", "GINI", "KS"):
+            self.assertIn(col, results.columns)
+
+        # Per-class and micro AUCs beat random, and GINI == 2*AUC - 1.
+        for _, row in results.iterrows():
+            self.assertGreater(row["AUC"], 0.5)
+            self.assertAlmostEqual(row["GINI"], 2 * row["AUC"] - 1, places=5)
+            self.assertGreaterEqual(row["KS"], 0.0)
+            self.assertLessEqual(row["KS"], 1.0)
+
+        # RawData carries per-class + micro fpr/tpr curves.
+        self.assertEqual(set(raw_data.fpr), {"0", "1", "2", "micro"})
+        self.assertEqual(set(raw_data.tpr), {"0", "1", "2", "micro"})
+
+    def test_multiclass_without_predict_proba_is_skipped(self):
+        # A model that can't produce a per-class probability matrix must skip,
+        # not crash. Shadow predict_proba on the estimator instance (predictions
+        # were already assigned in setUp, so nothing downstream needs it).
+        self.vm_model.model.predict_proba = None
+        with self.assertRaises(SkipTestError):
+            GINITable(self.vm_dataset, self.vm_model)

validmind/__version__.py

@@ -1 +1 @@
-__version__ = "2.13.6"
+__version__ = "2.13.7"

validmind/tests/model_validation/statsmodels/GINITable.py

@@ -7,8 +7,10 @@
 import numpy as np
 import pandas as pd
 from sklearn.metrics import roc_auc_score, roc_curve
+from sklearn.preprocessing import label_binarize
 
 from validmind import RawData, tags, tasks
+from validmind.errors import SkipTestError
 from validmind.vm_models import VMDataset, VMModel
 
 
@@ -58,12 +60,19 @@ def GINITable(dataset: VMDataset, model: VMModel) -> Tuple[pd.DataFrame, RawData
 
     - The GINI coefficient and KS statistic are both dependent on the AUC value. Therefore, any errors in the
     calculation of the latter will adversely impact the former metrics too.
-    - Mainly suited for binary classification models and may require modifications for effective application in
-    multi-class scenarios.
+    - For multiclass models the metrics are computed one-vs-rest (one row per class plus a micro-average row), which
+    requires per-class probabilities from the model's `predict_proba`. Models that cannot produce a full per-class
+    probability matrix (e.g. metadata-only models, or predictions supplied as a single precomputed probability column)
+    are skipped for the multiclass case rather than crashed.
     - The metrics used are threshold-dependent and may exhibit high variability based on the chosen cut-off points.
     - The test does not incorporate a method to efficiently handle missing or inefficiently processed data, which could
     lead to inaccuracies in the metrics if the data is not appropriately preprocessed.
     """
+    classes = np.unique(dataset.y)
+
+    if len(classes) > 2:
+        return _multiclass_gini_table(model, dataset, classes)
+
     y_true = np.ravel(dataset.y)  # Flatten y_true to make it one-dimensional
     y_prob = dataset.y_prob(model)
     y_true = np.array(y_true, dtype=float)
@@ -87,3 +96,81 @@ def GINITable(dataset: VMDataset, model: VMModel) -> Tuple[pd.DataFrame, RawData
         model=model.input_id,
         dataset=dataset.input_id,
     )
+
+
+def _multiclass_gini_table(
+    model: VMModel, dataset: VMDataset, classes: np.ndarray
+) -> Tuple[pd.DataFrame, RawData]:
+    """One-vs-rest AUC/GINI/KS for a multiclass model.
+
+    Needs the full per-class probability matrix, which the stored single
+    probability column cannot provide, so we ask the underlying estimator for
+    it directly. Models without a usable ``predict_proba`` (metadata-only,
+    precomputed single-column probabilities) are skipped rather than crashed.
+    """
+    # The VMModel wrapper's y_prob is binary-only (positive-class column only),
+    # so reach the underlying estimator for the full per-class probability matrix.
+    raw_model = getattr(model, "model", None)
+    proba_fn = getattr(raw_model, "predict_proba", None)
+    if not callable(proba_fn):
+        raise SkipTestError(
+            "Multiclass GINITable requires per-class probabilities from the "
+            "underlying model's predict_proba, which is not available for this "
+            "model (e.g. metadata-only / precomputed predictions). Skipping."
+        )
+    try:
+        y_prob = np.asarray(proba_fn(dataset.x_df()))
+    except Exception as e:
+        raise SkipTestError(
+            "Multiclass GINITable could not compute per-class probabilities "
+            f"({type(e).__name__}). Skipping."
+        ) from e
+
+    n_classes = len(classes)
+    if y_prob.ndim != 2 or y_prob.shape[1] != n_classes:
+        raise SkipTestError(
+            "Multiclass GINITable requires a per-class probability matrix with "
+            f"one column per class (got shape {getattr(y_prob, 'shape', None)} "
+            f"for {n_classes} classes). Skipping."
+        )
+
+    # One-hot the true labels in the same class order predict_proba columns use
+    # (sklearn orders predict_proba columns by sorted class label == np.unique).
+    y_true = dataset.y.flatten()
+    y_bin = label_binarize(y_true, classes=classes)
+
+    rows = []
+    raw_fpr = {}
+    raw_tpr = {}
+    for i, cls in enumerate(classes):
+        fpr, tpr, _ = roc_curve(y_bin[:, i], y_prob[:, i])
+        auc = roc_auc_score(y_bin[:, i], y_prob[:, i])
+        key = str(cls)
+        raw_fpr[key] = fpr
+        raw_tpr[key] = tpr
+        rows.append(
+            {"Class": key, "AUC": auc, "GINI": 2 * auc - 1, "KS": max(tpr - fpr)}
+        )
+
+    # Micro-average across all one-vs-rest decisions.
+    micro_fpr, micro_tpr, _ = roc_curve(y_bin.ravel(), y_prob.ravel())
+    micro_auc = roc_auc_score(y_bin, y_prob, average="micro", multi_class="ovr")
+    raw_fpr["micro"] = micro_fpr
+    raw_tpr["micro"] = micro_tpr
+    rows.append(
+        {
+            "Class": "micro",
+            "AUC": micro_auc,
+            "GINI": 2 * micro_auc - 1,
+            "KS": max(micro_tpr - micro_fpr),
+        }
+    )
+
+    return pd.DataFrame(rows), RawData(
+        fpr=raw_fpr,
+        tpr=raw_tpr,
+        y_true=y_true,
+        y_prob=y_prob,
+        model=model.input_id,
+        dataset=dataset.input_id,
+    )