From 4bfbc5acf62ce461bbf1e634315e03252c7836d9 Mon Sep 17 00:00:00 2001 From: ValidMind Support Agent Date: Tue, 7 Jul 2026 14:53:20 -0700 Subject: [PATCH 1/2] fix: WeakspotsDiagnosis handles multiclass/non-{0,1} labels (ZD-704) WeakspotsDiagnosis computed precision/recall/f1 per feature-bin slice with sklearn's binary defaults (average='binary', pos_label=1), which raised for multiclass models and for binary labels not containing 1 (e.g. {0, 4} -> 'pos_label=1 is not a valid label'). Decide the averaging once from the global label set (union of target and prediction columns of both datasets) and bind the default precision/recall/f1 metrics via functools.partial -- macro for multiclass, pos_label for non-standard binary, untouched for conventional {0, 1}. Mirrors the MinimumF1Score fix (PR #529). Adds regression tests. Co-Authored-By: Claude Fable 5 --- .../sklearn/test_WeakspotsDiagnosis.py | 126 ++++++++++++++++++ .../sklearn/WeakspotsDiagnosis.py | 74 ++++++++-- 2 files changed, 191 insertions(+), 9 deletions(-) diff --git a/tests/unit_tests/model_validation/sklearn/test_WeakspotsDiagnosis.py b/tests/unit_tests/model_validation/sklearn/test_WeakspotsDiagnosis.py index ff29cc113..0be8db88a 100644 --- a/tests/unit_tests/model_validation/sklearn/test_WeakspotsDiagnosis.py +++ b/tests/unit_tests/model_validation/sklearn/test_WeakspotsDiagnosis.py @@ -1,10 +1,71 @@ +import functools import unittest +import numpy as np +import pandas as pd +from sklearn import metrics +from sklearn.linear_model import LogisticRegression + +import validmind as vm from validmind.tests.model_validation.sklearn.WeakspotsDiagnosis import ( + WeakspotsDiagnosis, + _averaged_default_metrics, _prepare_metrics_and_thresholds, ) +def _train_test_datasets(input_id, labels, seed=0, n=140): + """Build train/test VMDatasets and a fitted model for a given label set. + + Two numeric features are generated so ``WeakspotsDiagnosis`` has columns to + bin, with one feature correlated with the class so individual bins carry + only a subset of the classes (which is what makes per-slice averaging + crash). Predictions are injected verbatim so the true/predicted label sets + are controlled exactly and never accidentally collapse to {0, 1}. + """ + frames = [] + for offset in (0, 100): + rng = np.random.default_rng(seed + offset) + y = rng.choice(labels, size=n) + frames.append( + pd.DataFrame( + { + "f1": y + rng.normal(0, 0.4, n), + "f2": rng.normal(0, 1, n), + "target": y, + } + ) + ) + train_df, test_df = frames + + train_ds = vm.init_dataset( + input_id=f"{input_id}_train", + dataset=train_df, + target_column="target", + __log=False, + ) + test_ds = vm.init_dataset( + input_id=f"{input_id}_test", + dataset=test_df, + target_column="target", + __log=False, + ) + + model = LogisticRegression(max_iter=2000) + model.fit(train_df[["f1", "f2"]].to_numpy(), train_df["target"].to_numpy()) + vm_model = vm.init_model(input_id=f"{input_id}_model", model=model, __log=False) + + train_ds.assign_predictions( + model=vm_model, + prediction_values=model.predict(train_df[["f1", "f2"]].to_numpy()), + ) + test_ds.assign_predictions( + model=vm_model, + prediction_values=model.predict(test_df[["f1", "f2"]].to_numpy()), + ) + return train_ds, test_ds, vm_model + + class TestWeakspotsDiagnosisThresholds(unittest.TestCase): def test_partial_thresholds_use_defaults_for_plotting(self): _, plot_thresholds, pass_thresholds = _prepare_metrics_and_thresholds( @@ -27,5 +88,70 @@ def test_partial_thresholds_subset_for_pass_fail(self): self.assertEqual(set(pass_thresholds.keys()), {"Accuracy", "F1"}) +class TestWeakspotsDefaultMetricAveraging(unittest.TestCase): + """Unit tests for the averaging strategy selected from the global labels. + + Regression tests for ZD-704: sklearn's precision/recall/f1 default to + ``average="binary", pos_label=1`` which raises for multiclass labels and for + binary labels that don't contain 1 (e.g. {0, 4}). + """ + + def test_multiclass_labels_use_macro(self): + labels = np.array([0, 2, 4]) + bound = _averaged_default_metrics(labels) + + for name in ("precision", "recall", "f1"): + self.assertIsInstance(bound[name], functools.partial) + self.assertEqual(bound[name].keywords["average"], "macro") + self.assertEqual(bound[name].keywords["zero_division"], 0) + self.assertTrue(np.array_equal(bound[name].keywords["labels"], labels)) + # accuracy takes no averaging kwargs and must be left alone + self.assertIs(bound["accuracy"], metrics.accuracy_score) + + def test_non_standard_binary_uses_pos_label(self): + # Two classes, neither of which is 1 -- the customer's exact case. + bound = _averaged_default_metrics(np.array([0, 4])) + + for name in ("precision", "recall", "f1"): + self.assertIsInstance(bound[name], functools.partial) + self.assertEqual(bound[name].keywords, {"pos_label": 4}) + self.assertIs(bound["accuracy"], metrics.accuracy_score) + + def test_conventional_binary_left_unchanged(self): + bound = _averaged_default_metrics(np.array([0, 1])) + + self.assertIs(bound["precision"], metrics.precision_score) + self.assertIs(bound["recall"], metrics.recall_score) + self.assertIs(bound["f1"], metrics.f1_score) + self.assertIs(bound["accuracy"], metrics.accuracy_score) + + +class TestWeakspotsDiagnosisMulticlass(unittest.TestCase): + """End-to-end regression tests for ZD-704. + + On unpatched code these raise ``ValueError`` per feature-bin slice; the fix + decides the averaging once from the global label set so the test runs. + """ + + def test_multiclass_non_contiguous_labels(self): + train_ds, test_ds, model = _train_test_datasets("wsd_multi", [0, 2, 4], seed=1) + # Would raise "Target is multiclass but average='binary'" / "pos_label=1 + # is not a valid label" on a per-slice basis before the fix. + result = WeakspotsDiagnosis(datasets=[train_ds, test_ds], model=model) + self.assertGreater(len(result), 1) + + def test_binary_labels_without_one(self): + train_ds, test_ds, model = _train_test_datasets("wsd_binary04", [0, 4], seed=2) + # Would raise "pos_label=1 is not a valid label. It should be one of + # [0, 4]" before the fix -- the customer's reported error. + result = WeakspotsDiagnosis(datasets=[train_ds, test_ds], model=model) + self.assertGreater(len(result), 1) + + def test_conventional_binary_still_runs(self): + train_ds, test_ds, model = _train_test_datasets("wsd_binary01", [0, 1], seed=3) + result = WeakspotsDiagnosis(datasets=[train_ds, test_ds], model=model) + self.assertGreater(len(result), 1) + + if __name__ == "__main__": unittest.main() diff --git a/validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py b/validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py index c92fee871..6e287fb89 100644 --- a/validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py +++ b/validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py @@ -2,9 +2,11 @@ # Refer to the LICENSE file in the root of this repository for details. # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial +import functools from typing import Callable, Dict, List, Optional, Tuple import matplotlib.pyplot as plt +import numpy as np import pandas as pd import plotly.graph_objects as go import seaborn as sns @@ -32,9 +34,37 @@ def _normalize_dict_keys(d: Dict) -> Dict: return {k.title(): v for k, v in d.items()} +def _averaged_default_metrics(labels: np.ndarray) -> Dict[str, Callable]: + """Bind the default precision/recall/f1 metrics to a suitable averaging mode. + + scikit-learn's precision/recall/f1 default to ``average="binary"`` with + ``pos_label=1``, which raises for multiclass labels and for binary labels + that do not include ``1`` (e.g. ``{0, 4}``). Following the MinimumF1Score fix + (PR #529), the averaging is decided once from the global label set so the + per-slice scores stay comparable: + + - multiclass (>2 labels): macro averaging over the global ``labels`` with + ``zero_division=0`` to suppress undefined-metric warnings for slices that + happen to lack some classes; + - binary but ``1`` is not a label: use the larger label as ``pos_label``; + - conventional binary ({0, 1}): leave scikit-learn's defaults untouched. + """ + averaged = dict(DEFAULT_METRICS) + if len(labels) > 2: + kwargs = {"average": "macro", "labels": labels, "zero_division": 0} + elif 1 not in labels: + kwargs = {"pos_label": labels.max()} + else: + return averaged + for name in ("precision", "recall", "f1"): + averaged[name] = functools.partial(averaged[name], **kwargs) + return averaged + + def _prepare_metrics_and_thresholds( metrics: Optional[Dict[str, Callable]], thresholds: Optional[Dict[str, float]], + labels: Optional[np.ndarray] = None, ) -> Tuple[Dict[str, Callable], Dict[str, float], Dict[str, float]]: """ Prepare metrics and threshold dicts for plotting and pass/fail checks. @@ -43,7 +73,15 @@ def _prepare_metrics_and_thresholds( Plotting uses default thresholds for any metric without an explicit value so charts always show a reference line; pass/fail uses only the user-provided thresholds when a custom dict is supplied. + + When the default metrics are used, ``labels`` (the global label set) selects + the averaging mode for precision/recall/f1 so multiclass and non-{0, 1} + binary models do not crash. Custom metric callables own their own kwargs and + are left untouched. """ + if metrics is None and labels is not None: + metrics = _averaged_default_metrics(labels) + normalized_metrics = _normalize_dict_keys(metrics or DEFAULT_METRICS) default_thresholds = _normalize_dict_keys(DEFAULT_THRESHOLDS) @@ -240,6 +278,8 @@ def WeakspotsDiagnosis( data types only. - Despite its usefulness in highlighting problematic regions, the test does not offer direct suggestions for model improvement. + - For multiclass models the default precision/recall/f1 metrics are macro-averaged over the global label set; for + binary models whose labels do not include ``1`` (e.g. ``{0, 4}``) the larger label is used as the positive class. """ feature_columns = features_columns or datasets[0].feature_columns numeric_and_categorical_columns = ( @@ -260,8 +300,32 @@ def WeakspotsDiagnosis( "Column(s) provided in features_columns do not exist in the dataset" ) + df_1 = datasets[0]._df[ + feature_columns + + [datasets[0].target_column, datasets[0].prediction_column(model)] + ] + df_2 = datasets[1]._df[ + feature_columns + + [datasets[1].target_column, datasets[1].prediction_column(model)] + ] + + # Decide the precision/recall/f1 averaging once from the labels sklearn will + # actually see -- the union of the target and prediction columns of both + # datasets. Deciding per slice would silently mix binary and macro semantics + # across bars of the same chart and still break on {0, 4}-style slices. + labels = np.unique( + np.concatenate( + [ + df_1[datasets[0].target_column].values, + df_1[datasets[0].prediction_column(model)].values, + df_2[datasets[1].target_column].values, + df_2[datasets[1].prediction_column(model)].values, + ] + ) + ) + metrics, plot_thresholds, pass_thresholds = _prepare_metrics_and_thresholds( - metrics, thresholds + metrics, thresholds, labels ) results_headers = ["Slice", "Number of Records", "Feature"] @@ -270,14 +334,6 @@ def WeakspotsDiagnosis( figures = [] passed = True - df_1 = datasets[0]._df[ - feature_columns - + [datasets[0].target_column, datasets[0].prediction_column(model)] - ] - df_2 = datasets[1]._df[ - feature_columns - + [datasets[1].target_column, datasets[1].prediction_column(model)] - ] results_1 = pd.DataFrame() results_2 = pd.DataFrame() for feature in feature_columns: From 73a8105e084388fc05085eadd4497a10b764163d Mon Sep 17 00:00:00 2001 From: ValidMind Support Agent Date: Tue, 7 Jul 2026 14:53:28 -0700 Subject: [PATCH 2/2] fix: harden Overfit/Robustness diagnosis f1/precision/recall (ZD-704) OverfitDiagnosis and RobustnessDiagnosis compute f1/precision/recall with sklearn's binary defaults when the user selects those metrics, crashing on multiclass models and on binary labels not containing 1. Resolve the averaging once from the global label set (union of y_true and predictions), the same guard applied to WeakspotsDiagnosis. Non-prf metrics (auc, accuracy, regression) are untouched. Adds regression tests. Co-Authored-By: Claude Fable 5 --- .../sklearn/test_OverfitDiagnosis.py | 121 ++++++++++++++++++ .../sklearn/test_RobustnessDiagnosis.py | 121 ++++++++++++++++++ .../sklearn/OverfitDiagnosis.py | 44 ++++++- .../sklearn/RobustnessDiagnosis.py | 54 +++++++- 4 files changed, 332 insertions(+), 8 deletions(-) create mode 100644 tests/unit_tests/model_validation/sklearn/test_OverfitDiagnosis.py create mode 100644 tests/unit_tests/model_validation/sklearn/test_RobustnessDiagnosis.py diff --git a/tests/unit_tests/model_validation/sklearn/test_OverfitDiagnosis.py b/tests/unit_tests/model_validation/sklearn/test_OverfitDiagnosis.py new file mode 100644 index 000000000..e9a8390c3 --- /dev/null +++ b/tests/unit_tests/model_validation/sklearn/test_OverfitDiagnosis.py @@ -0,0 +1,121 @@ +import functools +import unittest + +import numpy as np +import pandas as pd +from sklearn import metrics +from sklearn.linear_model import LogisticRegression + +import validmind as vm +from validmind.tests.model_validation.sklearn.OverfitDiagnosis import ( + OverfitDiagnosis, + _classification_metric_fn, +) + + +def _classification_datasets(input_id, labels, seed=0, n=160): + """Build train/test VMDatasets with a fitted classifier. + + Predictions and probabilities are computed by the model (not injected) so + ``OverfitDiagnosis`` detects a classification task via the probability + column. One feature is correlated with the class so feature bins carry a + subset of the classes. + """ + frames = [] + for offset in (0, 100): + rng = np.random.default_rng(seed + offset) + y = rng.choice(labels, size=n) + frames.append( + pd.DataFrame( + { + "f1": y + rng.normal(0, 0.4, n), + "f2": rng.normal(0, 1, n), + "target": y, + } + ) + ) + train_df, test_df = frames + + train_ds = vm.init_dataset( + input_id=f"{input_id}_train", + dataset=train_df, + target_column="target", + __log=False, + ) + test_ds = vm.init_dataset( + input_id=f"{input_id}_test", + dataset=test_df, + target_column="target", + __log=False, + ) + + model = LogisticRegression(max_iter=2000) + model.fit(train_df[["f1", "f2"]].to_numpy(), train_df["target"].to_numpy()) + vm_model = vm.init_model(input_id=f"{input_id}_model", model=model, __log=False) + + train_ds.assign_predictions(model=vm_model) + test_ds.assign_predictions(model=vm_model) + return train_ds, test_ds, vm_model + + +class TestOverfitClassificationMetricFn(unittest.TestCase): + """Unit tests for the averaging strategy selected from the global labels.""" + + def test_multiclass_labels_use_macro(self): + labels = np.array([0, 2, 4]) + fn = _classification_metric_fn("f1", labels) + + self.assertIsInstance(fn, functools.partial) + self.assertIs(fn.func, metrics.f1_score) + self.assertEqual(fn.keywords["average"], "macro") + self.assertEqual(fn.keywords["zero_division"], 0) + self.assertTrue(np.array_equal(fn.keywords["labels"], labels)) + + def test_non_standard_binary_uses_pos_label(self): + fn = _classification_metric_fn("precision", np.array([0, 4])) + + self.assertIsInstance(fn, functools.partial) + self.assertIs(fn.func, metrics.precision_score) + self.assertEqual(fn.keywords, {"pos_label": 4}) + + def test_conventional_binary_left_unchanged(self): + self.assertIs( + _classification_metric_fn("recall", np.array([0, 1])), metrics.recall_score + ) + + def test_non_prf_metrics_left_unchanged(self): + # auc/accuracy never take averaging kwargs and must not be wrapped. + self.assertIs( + _classification_metric_fn("auc", np.array([0, 2, 4])), + metrics.roc_auc_score, + ) + self.assertIs( + _classification_metric_fn("accuracy", np.array([0, 2, 4])), + metrics.accuracy_score, + ) + + +class TestOverfitDiagnosisMulticlass(unittest.TestCase): + """Regression tests for ZD-704 sibling exposure (explicit f1 selection).""" + + def test_multiclass_f1(self): + train_ds, test_ds, model = _classification_datasets( + "ovf_multi", [0, 2, 4], seed=1 + ) + result = OverfitDiagnosis( + model=model, datasets=[train_ds, test_ds], metric="f1" + ) + self.assertIn("Overfit Diagnosis", result[0]) + + def test_binary_f1_without_one(self): + train_ds, test_ds, model = _classification_datasets( + "ovf_binary04", [0, 4], seed=2 + ) + result = OverfitDiagnosis( + model=model, datasets=[train_ds, test_ds], metric="f1" + ) + self.assertIn("Overfit Diagnosis", result[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit_tests/model_validation/sklearn/test_RobustnessDiagnosis.py b/tests/unit_tests/model_validation/sklearn/test_RobustnessDiagnosis.py new file mode 100644 index 000000000..233ab532a --- /dev/null +++ b/tests/unit_tests/model_validation/sklearn/test_RobustnessDiagnosis.py @@ -0,0 +1,121 @@ +import functools +import unittest + +import numpy as np +import pandas as pd +from sklearn import metrics +from sklearn.linear_model import LogisticRegression + +import validmind as vm +from validmind.tests.model_validation.sklearn.RobustnessDiagnosis import ( + RobustnessDiagnosis, + _classification_metric_fn, +) + + +def _classification_datasets(input_id, labels, seed=0, n=160): + """Build train/test VMDatasets with a fitted classifier. + + ``RobustnessDiagnosis`` perturbs the numeric features and calls + ``model.predict`` on the noisy data, so a genuinely fitted model is needed + (predictions cannot be injected). Two numeric features are provided for the + noise to act on. + """ + frames = [] + for offset in (0, 100): + rng = np.random.default_rng(seed + offset) + y = rng.choice(labels, size=n) + frames.append( + pd.DataFrame( + { + "f1": y + rng.normal(0, 0.4, n), + "f2": rng.normal(0, 1, n), + "target": y, + } + ) + ) + train_df, test_df = frames + + train_ds = vm.init_dataset( + input_id=f"{input_id}_train", + dataset=train_df, + target_column="target", + __log=False, + ) + test_ds = vm.init_dataset( + input_id=f"{input_id}_test", + dataset=test_df, + target_column="target", + __log=False, + ) + + model = LogisticRegression(max_iter=2000) + model.fit(train_df[["f1", "f2"]].to_numpy(), train_df["target"].to_numpy()) + vm_model = vm.init_model(input_id=f"{input_id}_model", model=model, __log=False) + + train_ds.assign_predictions(model=vm_model) + test_ds.assign_predictions(model=vm_model) + return train_ds, test_ds, vm_model + + +class TestRobustnessClassificationMetricFn(unittest.TestCase): + """Unit tests for the averaging strategy selected from the global labels.""" + + def test_multiclass_labels_use_macro(self): + labels = np.array([0, 2, 4]) + fn = _classification_metric_fn("f1", labels) + + self.assertIsInstance(fn, functools.partial) + self.assertIs(fn.func, metrics.f1_score) + self.assertEqual(fn.keywords["average"], "macro") + self.assertEqual(fn.keywords["zero_division"], 0) + self.assertTrue(np.array_equal(fn.keywords["labels"], labels)) + + def test_non_standard_binary_uses_pos_label(self): + fn = _classification_metric_fn("recall", np.array([0, 4])) + + self.assertIsInstance(fn, functools.partial) + self.assertIs(fn.func, metrics.recall_score) + self.assertEqual(fn.keywords, {"pos_label": 4}) + + def test_conventional_binary_left_unchanged(self): + self.assertIs( + _classification_metric_fn("precision", np.array([0, 1])), + metrics.precision_score, + ) + + def test_non_prf_metrics_left_unchanged(self): + self.assertIs( + _classification_metric_fn("auc", np.array([0, 2, 4])), + metrics.roc_auc_score, + ) + self.assertIs( + _classification_metric_fn("accuracy", np.array([0, 2, 4])), + metrics.accuracy_score, + ) + + +class TestRobustnessDiagnosisMulticlass(unittest.TestCase): + """Regression tests for ZD-704 sibling exposure (explicit f1 selection).""" + + def test_multiclass_f1(self): + train_ds, test_ds, model = _classification_datasets( + "rbd_multi", [0, 2, 4], seed=1 + ) + result = RobustnessDiagnosis( + datasets=[train_ds, test_ds], model=model, metric="f1" + ) + self.assertIsInstance(result[0], pd.DataFrame) + + def test_binary_f1_without_one(self): + train_ds, test_ds, model = _classification_datasets( + "rbd_binary04", [0, 4], seed=2 + ) + result = RobustnessDiagnosis( + datasets=[train_ds, test_ds], model=model, metric="f1" + ) + self.assertIsInstance(result[0], pd.DataFrame) + + +if __name__ == "__main__": + unittest.main() diff --git a/validmind/tests/model_validation/sklearn/OverfitDiagnosis.py b/validmind/tests/model_validation/sklearn/OverfitDiagnosis.py index ecff1006d..3ce6ea16b 100644 --- a/validmind/tests/model_validation/sklearn/OverfitDiagnosis.py +++ b/validmind/tests/model_validation/sklearn/OverfitDiagnosis.py @@ -2,7 +2,8 @@ # Refer to the LICENSE file in the root of this repository for details. # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial -from typing import Dict, List, Tuple +import functools +from typing import Callable, Dict, List, Tuple import matplotlib.pyplot as plt import numpy as np @@ -64,6 +65,25 @@ } +def _classification_metric_fn(metric: str, labels: np.ndarray) -> Callable: + """Resolve the metric function, binding the averaging mode when needed. + + scikit-learn's precision/recall/f1 default to ``average="binary"`` with + ``pos_label=1``, which raises for multiclass labels and for binary labels + that do not include ``1`` (e.g. ``{0, 4}``). Following the MinimumF1Score fix + (PR #529), decide the averaging once from the global label set. Every other + metric (accuracy, auc, ...) is returned unchanged. + """ + fn = PERFORMANCE_METRICS[metric]["function"] + if metric not in ("f1", "precision", "recall"): + return fn + if len(labels) > 2: + return functools.partial(fn, average="macro", labels=labels, zero_division=0) + if 1 not in labels: + return functools.partial(fn, pos_label=labels.max()) + return fn + + def _prepare_results( results_train: dict, results_test: dict, metric: str ) -> pd.DataFrame: @@ -95,6 +115,7 @@ def _compute_metrics( pred_column: str, feature_column: str, metric: str, + metric_func: Callable, is_classification: bool, ) -> None: results["slice"].append(str(region)) @@ -106,7 +127,6 @@ def _compute_metrics( results[metric].append(0) return - metric_func = PERFORMANCE_METRICS[metric]["function"] y_true = df_region[target_column].values # AUC requires probability scores @@ -253,6 +273,24 @@ def OverfitDiagnosis( train_df[prob_column] = datasets[0].y_prob(model) test_df[prob_column] = datasets[1].y_prob(model) + # Decide the precision/recall/f1 averaging once from the labels sklearn will + # actually see -- the union of the target and prediction columns across both + # datasets -- so multiclass and non-{0, 1} binary models do not crash. + if is_classification: + labels = np.unique( + np.concatenate( + [ + train_df[datasets[0].target_column].values, + train_df[pred_column].values, + test_df[datasets[1].target_column].values, + test_df[pred_column].values, + ] + ) + ) + metric_func = _classification_metric_fn(metric, labels) + else: + metric_func = PERFORMANCE_METRICS[metric]["function"] + test_results = [] figures = [] results_headers = ["slice", "shape", "feature", metric] @@ -276,6 +314,7 @@ def OverfitDiagnosis( prob_column=prob_column, pred_column=pred_column, metric=metric, + metric_func=metric_func, is_classification=is_classification, ) df_test_region = test_df[ @@ -291,6 +330,7 @@ def OverfitDiagnosis( prob_column=prob_column, pred_column=pred_column, metric=metric, + metric_func=metric_func, is_classification=is_classification, ) diff --git a/validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py b/validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py index b7639583d..f5871b539 100644 --- a/validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py +++ b/validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py @@ -2,9 +2,10 @@ # Refer to the LICENSE file in the root of this repository for details. # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial +import functools from collections import defaultdict from operator import add -from typing import List, Tuple +from typing import Callable, List, Tuple import numpy as np import pandas as pd @@ -85,8 +86,31 @@ def _add_noise_std_dev( return noisy_values +def _classification_metric_fn(metric: str, labels: np.ndarray) -> Callable: + """Resolve the metric function, binding the averaging mode when needed. + + scikit-learn's precision/recall/f1 default to ``average="binary"`` with + ``pos_label=1``, which raises for multiclass labels and for binary labels + that do not include ``1`` (e.g. ``{0, 4}``). Following the MinimumF1Score fix + (PR #529), decide the averaging once from the global label set. Every other + metric (accuracy, auc, ...) is returned unchanged. + """ + fn = PERFORMANCE_METRICS[metric]["function"] + if metric not in ("f1", "precision", "recall"): + return fn + if len(labels) > 2: + return functools.partial(fn, average="macro", labels=labels, zero_division=0) + if 1 not in labels: + return functools.partial(fn, pos_label=labels.max()) + return fn + + def _compute_metric( - dataset: VMDataset, model: VMModel, X: pd.DataFrame, metric: str + dataset: VMDataset, + model: VMModel, + X: pd.DataFrame, + metric: str, + metric_func: Callable = None, ) -> float: if metric not in PERFORMANCE_METRICS: raise ValueError( @@ -100,7 +124,10 @@ def _compute_metric( y_proba = model.predict(X) return metrics.roc_auc_score(dataset.y, y_proba) - return PERFORMANCE_METRICS[metric]["function"](dataset.y, model.predict(X)) + if metric_func is None: + metric_func = PERFORMANCE_METRICS[metric]["function"] + + return metric_func(dataset.y, model.predict(X)) def _compute_gap(result: dict, metric: str) -> float: @@ -267,6 +294,19 @@ def RobustnessDiagnosis( else DEFAULT_REGRESSION_METRIC ) + # Decide the precision/recall/f1 averaging once from the unperturbed labels -- + # the union of y_true and the baseline predictions across datasets -- so + # multiclass and non-{0, 1} binary models do not crash. + metric_func = None + if metric in ("f1", "precision", "recall"): + labels = np.unique( + np.concatenate( + [np.asarray(dataset.y) for dataset in datasets] + + [np.asarray(model.predict(dataset.x_df())) for dataset in datasets] + ) + ) + metric_func = _classification_metric_fn(metric, labels) + results = [{} for _ in range(len(datasets))] # add baseline results (no perturbation) @@ -281,6 +321,7 @@ def RobustnessDiagnosis( model=model, X=dataset.x_df(), metric=metric, + metric_func=metric_func, ) ] result["Performance Decay"] = [0.0] @@ -307,6 +348,7 @@ def RobustnessDiagnosis( model=model, X=temp_df, metric=metric, + metric_func=metric_func, ) ) result["Performance Decay"].append(_compute_gap(result, metric)) @@ -325,9 +367,9 @@ def RobustnessDiagnosis( # rename perturbation size for baseline # Convert to object type first to avoid dtype incompatibility warning results_df["Perturbation Size"] = results_df["Perturbation Size"].astype(object) - results_df.loc[results_df["Perturbation Size"] == 0.0, "Perturbation Size"] = ( - "Baseline (0.0)" - ) + results_df.loc[ + results_df["Perturbation Size"] == 0.0, "Perturbation Size" + ] = "Baseline (0.0)" return ( results_df,