diff --git a/sentinelml/drift/psi.py b/sentinelml/drift/psi.py index ab57bf6..33a6d84 100644 --- a/sentinelml/drift/psi.py +++ b/sentinelml/drift/psi.py @@ -12,6 +12,13 @@ def calculate_psi(baseline: np.ndarray, current: np.ndarray, num_buckets: int = raise ValueError("epsilon must be finite and strictly between zero and one") baseline, current = DataSanitizer.sanitize_arrays(baseline, current) + # Large opposite-sign values can overflow percentile interpolation or + # linspace subtraction. Positive scaling preserves bucket membership. + scale = max(np.max(np.abs(baseline)), np.max(np.abs(current))) + if scale > 1e150: + baseline = baseline / scale + current = current / scale + quantiles = np.linspace(0, 100, num_buckets + 1) bins = np.unique(np.percentile(baseline, quantiles)) if len(bins) < 3: diff --git a/src/drift_detector.py b/src/drift_detector.py index e748b59..f3b27a2 100644 --- a/src/drift_detector.py +++ b/src/drift_detector.py @@ -37,6 +37,12 @@ def calculate_psi(expected, actual, bins=10): if np.min(expected) == np.max(expected) == np.min(actual) == np.max(actual): return 0.0 + # Scaling extreme values avoids overflow in quantile and bin interpolation. + scale = max(np.max(np.abs(expected)), np.max(np.abs(actual))) + if scale > 1e150: + expected = expected / scale + actual = actual / scale + # Quantile-based bins from reference distribution breakpoints = np.unique( np.quantile( diff --git a/tests/test_psi_extreme_values.py b/tests/test_psi_extreme_values.py new file mode 100644 index 0000000..00dbffe --- /dev/null +++ b/tests/test_psi_extreme_values.py @@ -0,0 +1,44 @@ +import numpy as np +import pytest + +from sentinelml.drift.psi import calculate_psi + + +def test_psi_distinguishes_extreme_opposite_constants(): + reference = np.full(5, 1e308) + current = np.full(5, -1e308) + with np.errstate(over="raise", invalid="raise"): + value = calculate_psi(reference, current) + assert value > 1.0 + assert np.isfinite(value) + + +def test_psi_extreme_magnitudes_match_rescaled_distributions(): + baseline = np.array([-1.0, -0.5, 0.0, 0.5, 1.0]) + current = np.array([-0.8, -0.1, 0.2, 0.7, 0.9]) + expected = calculate_psi(baseline, current, num_buckets=4) + with np.errstate(over="raise", invalid="raise"): + actual = calculate_psi(baseline * 1e308, current * 1e308, num_buckets=4) + assert actual == pytest.approx(expected) + + +def test_pipeline_psi_distinguishes_extreme_opposite_constants(): + from src.drift_detector import calculate_psi as pipeline_psi + + reference = np.full(5, 1e308) + current = np.full(5, -1e308) + with np.errstate(over="raise", invalid="raise"): + value = pipeline_psi(reference, current) + assert value > 1.0 + assert np.isfinite(value) + + +def test_pipeline_psi_matches_rescaled_distributions(): + from src.drift_detector import calculate_psi as pipeline_psi + + baseline = np.array([-1.0, -0.5, 0.0, 0.5, 1.0]) + current = np.array([-0.8, -0.1, 0.2, 0.7, 0.9]) + expected = pipeline_psi(baseline, current, bins=4) + with np.errstate(over="raise", invalid="raise"): + actual = pipeline_psi(baseline * 1e308, current * 1e308, bins=4) + assert actual == pytest.approx(expected)