diff --git a/pyproject.toml b/pyproject.toml index 33c1e0b62..4c8bc6cbe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "validmind" -version = "2.13.6" +version = "2.13.7" description = "ValidMind Library" readme = "README.pypi.md" requires-python = ">=3.9,<3.15" diff --git a/r/validmind/DESCRIPTION b/r/validmind/DESCRIPTION index 728809cd3..ada73751f 100644 --- a/r/validmind/DESCRIPTION +++ b/r/validmind/DESCRIPTION @@ -1,7 +1,7 @@ Package: validmind Type: Package Title: Interface to the 'ValidMind' Platform -Version: 2.13.6 +Version: 2.13.7 Authors@R: c(person("Andres", "Rodriguez", role = c("aut", "cre","cph"), email = "andres@validmind.ai")) Maintainer: Andres Rodriguez diff --git a/tests/unit_tests/model_validation/statsmodels/test_GINITable.py b/tests/unit_tests/model_validation/statsmodels/test_GINITable.py index c91814899..f16918f75 100644 --- a/tests/unit_tests/model_validation/statsmodels/test_GINITable.py +++ b/tests/unit_tests/model_validation/statsmodels/test_GINITable.py @@ -1,9 +1,11 @@ import unittest import pandas as pd import numpy as np +from sklearn.datasets import make_classification from sklearn.linear_model import LogisticRegression import validmind as vm from validmind import RawData +from validmind.errors import SkipTestError from validmind.tests.model_validation.statsmodels.GINITable import GINITable @@ -180,3 +182,61 @@ def test_metric_ranges(self): self.assertAlmostEqual( results["GINI"].iloc[0], 2 * results["AUC"].iloc[0] - 1, places=5 ) + + +class TestGINITableMulticlass(unittest.TestCase): + def setUp(self): + # 3-class target with real signal so per-class AUCs beat random. + X, y = make_classification( + n_samples=300, + n_features=5, + n_informative=4, + n_redundant=0, + n_classes=3, + n_clusters_per_class=1, + random_state=42, + ) + df = pd.DataFrame(X, columns=[f"f{i}" for i in range(5)]) + df["target"] = y + + model = LogisticRegression(max_iter=1000) + model.fit(X, y) + + self.vm_dataset = vm.init_dataset( + input_id="mc_dataset", dataset=df, target_column="target", __log=False + ) + self.vm_model = vm.init_model( + input_id="mc_model", model=model, __log=False + ) + self.vm_dataset.assign_predictions(self.vm_model) + + def test_multiclass_one_vs_rest_rows(self): + results, raw_data = GINITable(self.vm_dataset, self.vm_model) + + self.assertIsInstance(results, pd.DataFrame) + self.assertIsInstance(raw_data, RawData) + + # One row per class + a micro-average row. + self.assertEqual(set(results["Class"]), {"0", "1", "2", "micro"}) + self.assertEqual(len(results), 4) + for col in ("Class", "AUC", "GINI", "KS"): + self.assertIn(col, results.columns) + + # Per-class and micro AUCs beat random, and GINI == 2*AUC - 1. + for _, row in results.iterrows(): + self.assertGreater(row["AUC"], 0.5) + self.assertAlmostEqual(row["GINI"], 2 * row["AUC"] - 1, places=5) + self.assertGreaterEqual(row["KS"], 0.0) + self.assertLessEqual(row["KS"], 1.0) + + # RawData carries per-class + micro fpr/tpr curves. + self.assertEqual(set(raw_data.fpr), {"0", "1", "2", "micro"}) + self.assertEqual(set(raw_data.tpr), {"0", "1", "2", "micro"}) + + def test_multiclass_without_predict_proba_is_skipped(self): + # A model that can't produce a per-class probability matrix must skip, + # not crash. Shadow predict_proba on the estimator instance (predictions + # were already assigned in setUp, so nothing downstream needs it). + self.vm_model.model.predict_proba = None + with self.assertRaises(SkipTestError): + GINITable(self.vm_dataset, self.vm_model) diff --git a/uv.lock b/uv.lock index a74e875a4..1688bb619 100644 --- a/uv.lock +++ b/uv.lock @@ -11340,7 +11340,7 @@ wheels = [ [[package]] name = "validmind" -version = "2.13.6" +version = "2.13.7" source = { editable = "." } dependencies = [ { name = "aiohttp", extra = ["speedups"] }, diff --git a/validmind/__version__.py b/validmind/__version__.py index 4de2b4e35..800de89f9 100644 --- a/validmind/__version__.py +++ b/validmind/__version__.py @@ -1 +1 @@ -__version__ = "2.13.6" +__version__ = "2.13.7" diff --git a/validmind/tests/model_validation/statsmodels/GINITable.py b/validmind/tests/model_validation/statsmodels/GINITable.py index 8b8eea5aa..5aa01d205 100644 --- a/validmind/tests/model_validation/statsmodels/GINITable.py +++ b/validmind/tests/model_validation/statsmodels/GINITable.py @@ -7,8 +7,10 @@ import numpy as np import pandas as pd from sklearn.metrics import roc_auc_score, roc_curve +from sklearn.preprocessing import label_binarize from validmind import RawData, tags, tasks +from validmind.errors import SkipTestError from validmind.vm_models import VMDataset, VMModel @@ -58,12 +60,19 @@ def GINITable(dataset: VMDataset, model: VMModel) -> Tuple[pd.DataFrame, RawData - The GINI coefficient and KS statistic are both dependent on the AUC value. Therefore, any errors in the calculation of the latter will adversely impact the former metrics too. - - Mainly suited for binary classification models and may require modifications for effective application in - multi-class scenarios. + - For multiclass models the metrics are computed one-vs-rest (one row per class plus a micro-average row), which + requires per-class probabilities from the model's `predict_proba`. Models that cannot produce a full per-class + probability matrix (e.g. metadata-only models, or predictions supplied as a single precomputed probability column) + are skipped for the multiclass case rather than crashed. - The metrics used are threshold-dependent and may exhibit high variability based on the chosen cut-off points. - The test does not incorporate a method to efficiently handle missing or inefficiently processed data, which could lead to inaccuracies in the metrics if the data is not appropriately preprocessed. """ + classes = np.unique(dataset.y) + + if len(classes) > 2: + return _multiclass_gini_table(model, dataset, classes) + y_true = np.ravel(dataset.y) # Flatten y_true to make it one-dimensional y_prob = dataset.y_prob(model) y_true = np.array(y_true, dtype=float) @@ -87,3 +96,81 @@ def GINITable(dataset: VMDataset, model: VMModel) -> Tuple[pd.DataFrame, RawData model=model.input_id, dataset=dataset.input_id, ) + + +def _multiclass_gini_table( + model: VMModel, dataset: VMDataset, classes: np.ndarray +) -> Tuple[pd.DataFrame, RawData]: + """One-vs-rest AUC/GINI/KS for a multiclass model. + + Needs the full per-class probability matrix, which the stored single + probability column cannot provide, so we ask the underlying estimator for + it directly. Models without a usable ``predict_proba`` (metadata-only, + precomputed single-column probabilities) are skipped rather than crashed. + """ + # The VMModel wrapper's y_prob is binary-only (positive-class column only), + # so reach the underlying estimator for the full per-class probability matrix. + raw_model = getattr(model, "model", None) + proba_fn = getattr(raw_model, "predict_proba", None) + if not callable(proba_fn): + raise SkipTestError( + "Multiclass GINITable requires per-class probabilities from the " + "underlying model's predict_proba, which is not available for this " + "model (e.g. metadata-only / precomputed predictions). Skipping." + ) + try: + y_prob = np.asarray(proba_fn(dataset.x_df())) + except Exception as e: + raise SkipTestError( + "Multiclass GINITable could not compute per-class probabilities " + f"({type(e).__name__}). Skipping." + ) from e + + n_classes = len(classes) + if y_prob.ndim != 2 or y_prob.shape[1] != n_classes: + raise SkipTestError( + "Multiclass GINITable requires a per-class probability matrix with " + f"one column per class (got shape {getattr(y_prob, 'shape', None)} " + f"for {n_classes} classes). Skipping." + ) + + # One-hot the true labels in the same class order predict_proba columns use + # (sklearn orders predict_proba columns by sorted class label == np.unique). + y_true = dataset.y.flatten() + y_bin = label_binarize(y_true, classes=classes) + + rows = [] + raw_fpr = {} + raw_tpr = {} + for i, cls in enumerate(classes): + fpr, tpr, _ = roc_curve(y_bin[:, i], y_prob[:, i]) + auc = roc_auc_score(y_bin[:, i], y_prob[:, i]) + key = str(cls) + raw_fpr[key] = fpr + raw_tpr[key] = tpr + rows.append( + {"Class": key, "AUC": auc, "GINI": 2 * auc - 1, "KS": max(tpr - fpr)} + ) + + # Micro-average across all one-vs-rest decisions. + micro_fpr, micro_tpr, _ = roc_curve(y_bin.ravel(), y_prob.ravel()) + micro_auc = roc_auc_score(y_bin, y_prob, average="micro", multi_class="ovr") + raw_fpr["micro"] = micro_fpr + raw_tpr["micro"] = micro_tpr + rows.append( + { + "Class": "micro", + "AUC": micro_auc, + "GINI": 2 * micro_auc - 1, + "KS": max(micro_tpr - micro_fpr), + } + ) + + return pd.DataFrame(rows), RawData( + fpr=raw_fpr, + tpr=raw_tpr, + y_true=y_true, + y_prob=y_prob, + model=model.input_id, + dataset=dataset.input_id, + )