From 9e3b91b316767467902d9692154f61795f9d239e Mon Sep 17 00:00:00 2001 From: Ayush Tiwari <147244828+Ayushdevo@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:10:32 +0530 Subject: [PATCH 1/2] Avoid overflow in PSI for extreme finite samples --- sentinelml/drift/psi.py | 7 +++++++ tests/test_psi_extreme_values.py | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 tests/test_psi_extreme_values.py diff --git a/sentinelml/drift/psi.py b/sentinelml/drift/psi.py index ab57bf6..33a6d84 100644 --- a/sentinelml/drift/psi.py +++ b/sentinelml/drift/psi.py @@ -12,6 +12,13 @@ def calculate_psi(baseline: np.ndarray, current: np.ndarray, num_buckets: int = raise ValueError("epsilon must be finite and strictly between zero and one") baseline, current = DataSanitizer.sanitize_arrays(baseline, current) + # Large opposite-sign values can overflow percentile interpolation or + # linspace subtraction. Positive scaling preserves bucket membership. + scale = max(np.max(np.abs(baseline)), np.max(np.abs(current))) + if scale > 1e150: + baseline = baseline / scale + current = current / scale + quantiles = np.linspace(0, 100, num_buckets + 1) bins = np.unique(np.percentile(baseline, quantiles)) if len(bins) < 3: diff --git a/tests/test_psi_extreme_values.py b/tests/test_psi_extreme_values.py new file mode 100644 index 0000000..d080d52 --- /dev/null +++ b/tests/test_psi_extreme_values.py @@ -0,0 +1,22 @@ +import numpy as np +import pytest + +from sentinelml.drift.psi import calculate_psi + + +def test_psi_distinguishes_extreme_opposite_constants(): + reference = np.full(5, 1e308) + current = np.full(5, -1e308) + with np.errstate(over="raise", invalid="raise"): + value = calculate_psi(reference, current) + assert value > 1.0 + assert np.isfinite(value) + + +def test_psi_extreme_magnitudes_match_rescaled_distributions(): + baseline = np.array([-1.0, -0.5, 0.0, 0.5, 1.0]) + current = np.array([-0.8, -0.1, 0.2, 0.7, 0.9]) + expected = calculate_psi(baseline, current, num_buckets=4) + with np.errstate(over="raise", invalid="raise"): + actual = calculate_psi(baseline * 1e308, current * 1e308, num_buckets=4) + assert actual == pytest.approx(expected) From e8d0db7be5ad93341d626ce61ddf0e14f430345f Mon Sep 17 00:00:00 2001 From: Ayush Tiwari <147244828+Ayushdevo@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:12:23 +0530 Subject: [PATCH 2/2] Prevent extreme-value overflow in pipeline PSI --- src/drift_detector.py | 6 ++++++ tests/test_psi_extreme_values.py | 22 ++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/drift_detector.py b/src/drift_detector.py index e748b59..f3b27a2 100644 --- a/src/drift_detector.py +++ b/src/drift_detector.py @@ -37,6 +37,12 @@ def calculate_psi(expected, actual, bins=10): if np.min(expected) == np.max(expected) == np.min(actual) == np.max(actual): return 0.0 + # Scaling extreme values avoids overflow in quantile and bin interpolation. + scale = max(np.max(np.abs(expected)), np.max(np.abs(actual))) + if scale > 1e150: + expected = expected / scale + actual = actual / scale + # Quantile-based bins from reference distribution breakpoints = np.unique( np.quantile( diff --git a/tests/test_psi_extreme_values.py b/tests/test_psi_extreme_values.py index d080d52..00dbffe 100644 --- a/tests/test_psi_extreme_values.py +++ b/tests/test_psi_extreme_values.py @@ -20,3 +20,25 @@ def test_psi_extreme_magnitudes_match_rescaled_distributions(): with np.errstate(over="raise", invalid="raise"): actual = calculate_psi(baseline * 1e308, current * 1e308, num_buckets=4) assert actual == pytest.approx(expected) + + +def test_pipeline_psi_distinguishes_extreme_opposite_constants(): + from src.drift_detector import calculate_psi as pipeline_psi + + reference = np.full(5, 1e308) + current = np.full(5, -1e308) + with np.errstate(over="raise", invalid="raise"): + value = pipeline_psi(reference, current) + assert value > 1.0 + assert np.isfinite(value) + + +def test_pipeline_psi_matches_rescaled_distributions(): + from src.drift_detector import calculate_psi as pipeline_psi + + baseline = np.array([-1.0, -0.5, 0.0, 0.5, 1.0]) + current = np.array([-0.8, -0.1, 0.2, 0.7, 0.9]) + expected = pipeline_psi(baseline, current, bins=4) + with np.errstate(over="raise", invalid="raise"): + actual = pipeline_psi(baseline * 1e308, current * 1e308, bins=4) + assert actual == pytest.approx(expected)