fix: avoid overflow in NumericDiff for finite inputs

js/number.test.ts

@@ -0,0 +1,19 @@
+import { describe, expect, test } from "vitest";
+import { NumericDiff } from "./number";
+
+describe("NumericDiff finite extremes", () => {
+  test.each([
+    [Number.MAX_VALUE, Number.MAX_VALUE, 1],
+    [Number.MAX_VALUE, Number.MAX_VALUE / 2, 2 / 3],
+    [Number.MAX_VALUE, -Number.MAX_VALUE, 0],
+    [-Number.MAX_VALUE, -Number.MAX_VALUE / 2, 2 / 3],
+    [1e-320, 1e-320 / 2, 2 / 3],
+    [0, 0, 1],
+    [0, Number.MAX_VALUE, 0],
+    [10, 5, 2 / 3],
+  ])("output %s and expected %s", async (output, expected, score) => {
+    const result = await NumericDiff({ output, expected });
+    expect(Number.isFinite(result.score)).toBe(true);
+    expect(result.score).toBeCloseTo(score, 12);
+  });
+});

js/number.ts

@@ -11,11 +11,16 @@ export const NumericDiff: ScorerWithPartial<number, {}> = makePartial(
       throw new Error("NumericDiff requires an expected value");
     }
 
+    // Scale first so finite inputs cannot overflow the difference or sum.
+    const scale = Math.max(Math.abs(expected), Math.abs(output));
+    const expectedScaled = scale === 0 ? 0 : expected / scale;
+    const outputScaled = scale === 0 ? 0 : output / scale;
     const score =
-      output === 0 && expected === 0
+      scale === 0
         ? 1
         : 1 -
-          Math.abs(expected - output) / (Math.abs(expected) + Math.abs(output));
+          Math.abs(expectedScaled - outputScaled) /
+            (Math.abs(expectedScaled) + Math.abs(outputScaled));
 
     return {
       name: "NumericDiff",

py/autoevals/number.py

@@ -47,7 +47,10 @@ def _run_eval_sync(self, output, expected=None, **kwargs):
         if expected == 0 and output == 0:
             score = 1
         else:
-            score = 1 - abs(expected - output) / (abs(expected) + abs(output))
+            # Scale first so finite inputs cannot overflow the difference or sum.
+            scale = max(abs(expected), abs(output))
+            expected_scaled, output_scaled = expected / scale, output / scale
+            score = 1 - abs(expected_scaled - output_scaled) / (abs(expected_scaled) + abs(output_scaled))
         return Score(name=self._name(), score=score)
 
 

py/autoevals/test_number.py

@@ -0,0 +1,25 @@
+import math
+import sys
+
+import pytest
+
+from autoevals.number import NumericDiff
+
+
+@pytest.mark.parametrize(
+    "output,expected,score",
+    [
+        (sys.float_info.max, sys.float_info.max, 1),
+        (sys.float_info.max, sys.float_info.max / 2, 2 / 3),
+        (sys.float_info.max, -sys.float_info.max, 0),
+        (-sys.float_info.max, -sys.float_info.max / 2, 2 / 3),
+        (1e-320, 1e-320 / 2, 2 / 3),
+        (0, 0, 1),
+        (0, sys.float_info.max, 0),
+        (10, 5, 2 / 3),
+    ],
+)
+def test_numeric_diff_finite_extremes(output, expected, score):
+    result = NumericDiff()(output, expected).score
+    assert math.isfinite(result)
+    assert result == pytest.approx(score)