Fix empirical E-value audit status for configurable ceilings

rfoldsearch/experimental_calibrate.py

@@ -305,7 +305,7 @@ def fit_files(args, *, predicted=False):
               "n_chains": len(ids), "n_clusters": audit["n_clusters"],
               "target_database_fingerprint": target_digest,
               "calibration_context_fingerprint": context_digest,
-              "validation_status": "audited-e0-le-100",
+              "validation_status": f"audited-e0-le-{audit_ceiling:.17g}",
               "audit_raw_evalue_ceiling": audit_ceiling,
               "target_aware_gumbel": model,
               "empirical_calibration": calibration,

rfoldsearch/experimental_evalue.py

@@ -287,13 +287,42 @@ def apply_empirical(evalues, calibration):
     return float(y) if np.ndim(y) == 0 else y
 
 
+def validate_audit_metadata(params):
+    """Check the audited range, including files with the old fixed-100 label."""
+    status = params.get("validation_status", "")
+    prefix = "audited-e0-le-"
+    if not isinstance(status, str) or not status.startswith(prefix):
+        raise ValueError("empirical E-values lack a completed structural audit")
+    try:
+        labelled = float(status[len(prefix):])
+        ceiling = float(params.get("audit_raw_evalue_ceiling", labelled))
+    except (TypeError, ValueError):
+        raise ValueError("invalid empirical audit ceiling") from None
+    if any(not math.isfinite(value) or value < 100 for value in (labelled, ceiling)):
+        raise ValueError("empirical audit ceilings must be finite and at least 100")
+    # Older fitters always wrote 100 in the label, even when the explicit
+    # audit_raw_evalue_ceiling recorded a larger, completed audit.
+    if labelled != ceiling and labelled != 100:
+        raise ValueError("empirical audit label and ceiling disagree")
+    calibration = params["empirical_calibration"]
+    try:
+        fit_ceiling = float(calibration.get("fit_raw_evalue_ceiling", ceiling))
+        last_knot = float(calibration["raw_evalues"][-1])
+        fit_max = float(calibration.get("fit_raw_evalue_max", last_knot))
+    except (TypeError, ValueError):
+        raise ValueError("invalid empirical calibration range") from None
+    if (not math.isfinite(fit_ceiling) or not 0 < fit_ceiling <= ceiling
+            or any(not math.isfinite(value) or not 0 < value <= fit_ceiling
+                   for value in (last_knot, fit_max))):
+        raise ValueError("empirical calibration range exceeds its audit or fitting ceiling")
+
+
 def validate_parameters(params, n_clusters, target_fingerprint, context_fingerprint):
     if params.get("scheme") != SCHEME or params.get("score_definition") != "raw-sw":
         raise ValueError("unsupported experimental E-value scheme or score")
     if (params.get("target_database_fingerprint") != target_fingerprint
             or params.get("calibration_context_fingerprint") != context_fingerprint
-            or int(params.get("n_clusters", -1)) != n_clusters
-            or params.get("validation_status") != "audited-e0-le-100"):
+            or int(params.get("n_clusters", -1)) != n_clusters):
         raise ValueError("experimental E-values belong to another database or lack audit")
     model = params["target_aware_gumbel"]
     _, _, _, _, multiplicities = _model_arrays(model)
@@ -303,3 +332,4 @@ def validate_parameters(params, n_clusters, target_fingerprint, context_fingerpr
     if int(calibration.get("n_clusters", -1)) != n_clusters:
         raise ValueError("empirical map cluster count does not match database")
     apply_empirical(np.array([0.0, 1.0]), calibration)
+    validate_audit_metadata(params)

rfoldsearch/predicted_evalue.py

@@ -13,7 +13,6 @@ def validate_parameters(params, n_clusters, target_fingerprint,
     if (params.get("scheme") != SCHEME
             or params.get("score_definition") != "compbias20-raw-sw"
             or params.get("composition_window") != compbias.WINDOW
-            or params.get("validation_status") != "audited-e0-le-100"
             or params.get("target_database_fingerprint") != target_fingerprint
             or params.get("calibration_context_fingerprint") != context_fingerprint
             or int(params.get("n_clusters", -1)) != n_clusters):
@@ -27,6 +26,7 @@ def validate_parameters(params, n_clusters, target_fingerprint,
             or calibration.get("background_scaling") != "complete-eligible-clusters"):
         raise ValueError("predicted empirical calibration needs the full eligible background")
     core.apply_empirical(np.array([0.0, 1.0]), calibration)
+    core.validate_audit_metadata(params)
     fitted = np.asarray(params["composition_background"], dtype=float)
     if fitted.ndim != 1 or not len(fitted):
         raise ValueError("invalid predicted composition background")

tests/test_empirical_audit_metadata.py

@@ -0,0 +1,58 @@
+"""Reject unaudited ranges without invalidating historical parameter files."""
+import pytest
+
+from rfoldsearch.experimental_evalue import validate_audit_metadata
+
+
+@pytest.mark.parametrize("status,ceiling,fit_ceiling,fit_max,knots", [
+    ("audited-e0-le-100", None, 100, 99, [.1, 99]),
+    ("audited-e0-le-400", 400, 400, 399, [.1, 399]),
+    ("audited-e0-le-100", 400, 400, 399, [.1, 399]),
+    ("audited-e0-le-400.125", 400.125, 400.125, 400.1, [.1, 400.1]),
+    ("audited-e0-le-400", 400, 100, 99, [.1, 99]),
+])
+def test_supported_audit_ranges(status, ceiling, fit_ceiling, fit_max, knots):
+    params = {"validation_status": status, "empirical_calibration": {
+        "fit_raw_evalue_ceiling": fit_ceiling, "fit_raw_evalue_max": fit_max,
+        "raw_evalues": knots}}
+    if ceiling is not None:
+        params["audit_raw_evalue_ceiling"] = ceiling
+    validate_audit_metadata(params)
+
+
+@pytest.mark.parametrize("changes", [
+    {"validation_status": "structural-audit-not-recorded"},
+    {"validation_status": "audited-e0-le-nan"},
+    {"validation_status": "audited-e0-le-400junk"},
+    {"validation_status": None},
+    {"validation_status": "audited-e0-le-500"},
+    {"audit_raw_evalue_ceiling": 100},
+    {"audit_raw_evalue_ceiling": 500},
+    {"audit_raw_evalue_ceiling": float("inf")},
+    {"audit_raw_evalue_ceiling": 99},
+    {"audit_raw_evalue_ceiling": None},
+])
+def test_invalid_audit_ranges(changes):
+    params = {"validation_status": "audited-e0-le-400",
+              "audit_raw_evalue_ceiling": 400,
+              "empirical_calibration": {"fit_raw_evalue_ceiling": 400,
+                  "fit_raw_evalue_max": 399, "raw_evalues": [.1, 399]}}
+    params.update(changes)
+    with pytest.raises(ValueError):
+        validate_audit_metadata(params)
+
+
+@pytest.mark.parametrize("changes", [
+    {"fit_raw_evalue_ceiling": 500},
+    {"fit_raw_evalue_ceiling": float("nan")},
+    {"fit_raw_evalue_max": 401},
+    {"fit_raw_evalue_max": float("inf")},
+    {"raw_evalues": [.1, 401]},
+])
+def test_calibration_cannot_exceed_audited_range(changes):
+    calibration = {"fit_raw_evalue_ceiling": 400,
+                   "fit_raw_evalue_max": 399, "raw_evalues": [.1, 399]}
+    calibration.update(changes)
+    with pytest.raises(ValueError, match="calibration range"):
+        validate_audit_metadata({"validation_status": "audited-e0-le-400",
+            "audit_raw_evalue_ceiling": 400, "empirical_calibration": calibration})

tests/test_experimental_empirical_evalue.py

@@ -90,7 +90,8 @@ def test_fitter_uses_clean_background_for_both_stages():
     assert uncensored["censoring"] == "none"
 
 
-def test_experimental_cli_fit_attach_and_search(tmp_path):
+@pytest.mark.parametrize("ceiling", [100, 400])
+def test_experimental_cli_fit_attach_and_search(tmp_path, ceiling):
     rng = np.random.default_rng(41)
     entries = add_coords([{"id": f"t{i}", "tokens": rng.integers(0, 20, 28 + i % 7).tolist(),
                            "cluster": f"c{i}"} for i in range(32)])
@@ -122,8 +123,12 @@ def test_experimental_cli_fit_attach_and_search(tmp_path):
     main(["fit-experimental-evalues", "--db", str(db), "--labels", str(labels),
           "--workdir", str(tmp_path / "fit"), "--output", str(params_path),
           "--scores", str(score_path), "--score-manifest", str(score_manifest),
-          "--tm-matrix", str(tm_path), "--tm-manifest", str(tm_manifest)])
+          "--tm-matrix", str(tm_path), "--tm-manifest", str(tm_manifest),
+          "--audit-ceiling", str(ceiling)])
     params = json.loads(params_path.read_text())
+    assert params["validation_status"] == f"audited-e0-le-{ceiling}"
+    assert params["audit_raw_evalue_ceiling"] == ceiling
+    assert params["empirical_calibration"]["fit_raw_evalue_ceiling"] == ceiling
     assert params["target_aware_gumbel"]["censoring"] == "left-censored-95th-percentile"
     assert params["empirical_calibration"]["upper_tail_rule"] == "anchored-power-cap"
     cluster_file = tmp_path / "clusters.tsv"
@@ -140,7 +145,12 @@ def test_experimental_cli_fit_attach_and_search(tmp_path):
         hit["raw_evalue"], params["empirical_calibration"]))
 
 
-def test_searcher_uses_matching_experimental_empirical_parameters(tmp_path):
+@pytest.mark.parametrize("status,ceiling", [
+    ("audited-e0-le-100", None),
+    ("audited-e0-le-400", 400),
+    ("audited-e0-le-100", 400),  # Parameters from the old fixed-label fitter.
+])
+def test_searcher_uses_matching_experimental_empirical_parameters(tmp_path, status, ceiling):
     entries = add_coords([{"id": f"t{i}", "tokens": [i % 5, (i + 1) % 5] * 15,
                            "cluster": f"c{i}"} for i in range(5)])
     target = database_fingerprint(entries, [e["cluster"] for e in entries], 80)
@@ -150,7 +160,7 @@ def test_searcher_uses_matching_experimental_empirical_parameters(tmp_path):
         "scheme": EV.SCHEME, "score_definition": "raw-sw", "n_clusters": 5,
         "target_database_fingerprint": target,
         "calibration_context_fingerprint": context,
-        "validation_status": "audited-e0-le-100",
+        "validation_status": status,
         "target_aware_gumbel": {
             "coefficients_mu_logbeta": [[8., 0.], [0., 0.], [0., 0.],
                                          [0., 0.], [0., 0.]],
@@ -163,6 +173,9 @@ def test_searcher_uses_matching_experimental_empirical_parameters(tmp_path):
             "low_tail_power_b": 1., "n_clusters": 5,
         },
     }
+    if ceiling is not None:
+        params["audit_raw_evalue_ceiling"] = ceiling
+        params["empirical_calibration"]["fit_raw_evalue_ceiling"] = ceiling
     db = tmp_path / "db.pt"
     torch.save({"n_codes": 80, "entries": entries, "n_clusters": 5,
                 "evalue_params": params}, db)
@@ -174,6 +187,11 @@ def test_searcher_uses_matching_experimental_empirical_parameters(tmp_path):
         params["target_aware_gumbel"]))
     assert hit["evalue"] == pytest.approx(EV.apply_empirical(
         hit["raw_evalue"], params["empirical_calibration"]))
+    bad_audit = tmp_path / "bad_audit.json"
+    bad_audit.write_text(json.dumps(dict(params,
+        validation_status="audited-e0-le-400", audit_raw_evalue_ceiling=100)))
+    with pytest.raises(ValueError, match="audit label and ceiling disagree"):
+        Searcher(str(db), evalue_params_path=str(bad_audit))
     params["target_database_fingerprint"] = "wrong"
     bad = tmp_path / "wrong.json"
     bad.write_text(json.dumps(params))

tests/test_predicted_empirical_evalue.py

@@ -65,7 +65,12 @@ def test_compressed_gumbel_likelihood_matches_uncompressed_fit():
         np.asarray(plain["coefficients_mu_logbeta"]), abs=1e-4)
 
 
-def test_search_uses_compbias_only_for_evalue_and_rejects_wrong_context(tmp_path):
+@pytest.mark.parametrize("status,ceiling", [
+    ("audited-e0-le-100", None),
+    ("audited-e0-le-400", 400),
+    ("audited-e0-le-100", 400),  # Parameters from the old fixed-label fitter.
+])
+def test_search_uses_compbias_only_for_evalue_and_rejects_wrong_context(tmp_path, status, ceiling):
     entries = add_coords([{"id": f"t{i}", "tokens": [i % 5, (i + 1) % 5] * 15,
                            "cluster": f"c{i}"} for i in range(5)])
     tokens = [entry["tokens"] for entry in entries]
@@ -79,7 +84,7 @@ def test_search_uses_compbias_only_for_evalue_and_rejects_wrong_context(tmp_path
         "composition_background": background.tolist(), "n_clusters": 5,
         "target_database_fingerprint": target,
         "calibration_context_fingerprint": context,
-        "validation_status": "audited-e0-le-100",
+        "validation_status": status,
         "target_aware_gumbel": {
             "coefficients_mu_logbeta": [[8., 0.], [0., 0.], [0., 0.],
                                          [0., 0.], [0., 0.]],
@@ -93,6 +98,9 @@ def test_search_uses_compbias_only_for_evalue_and_rejects_wrong_context(tmp_path
             "background_scaling": "complete-eligible-clusters",
         },
     }
+    if ceiling is not None:
+        params["audit_raw_evalue_ceiling"] = ceiling
+        params["empirical_calibration"]["fit_raw_evalue_ceiling"] = ceiling
     db = tmp_path / "db.pt"
     torch.save({"n_codes": 80, "entries": entries, "n_clusters": 5,
                 "evalue_params": params}, db)
@@ -111,14 +119,20 @@ def test_search_uses_compbias_only_for_evalue_and_rejects_wrong_context(tmp_path
         params["target_aware_gumbel"]))
     assert hit["evalue"] == pytest.approx(EV.apply_empirical(
         hit["raw_evalue"], params["empirical_calibration"]))
+    bad_audit = tmp_path / "bad_audit.json"
+    bad_audit.write_text(json.dumps(dict(params,
+        validation_status="audited-e0-le-400", audit_raw_evalue_ceiling=100)))
+    with pytest.raises(ValueError, match="audit label and ceiling disagree"):
+        Searcher(str(db), evalue_params_path=str(bad_audit))
     params["composition_background"][0] += .01
     bad = tmp_path / "wrong.json"
     bad.write_text(json.dumps(params))
     with pytest.raises(ValueError, match="composition background"):
         Searcher(str(db), evalue_params_path=str(bad))
 
 
-def test_fitting_cli_writes_attachable_predicted_parameters(tmp_path):
+@pytest.mark.parametrize("ceiling", [100, 400])
+def test_fitting_cli_writes_attachable_predicted_parameters(tmp_path, ceiling):
     rng = np.random.default_rng(30)
     entries = add_coords([{"id": f"t{i}", "tokens": rng.integers(0, 15, 25 + i % 8).tolist(),
                            "cluster": f"c{i}"} for i in range(40)])
@@ -163,14 +177,16 @@ def test_fitting_cli_writes_attachable_predicted_parameters(tmp_path):
           "--training-indices", str(training), "--calibration-indices", str(calibration),
           "--auxiliary-indices", str(auxiliary),
           "--workdir", str(tmp_path / "fit"), "--output", str(output),
-          "--audit-ceiling", "120",
+          "--audit-ceiling", str(ceiling),
           "--matrix", str(matrix_path), "--scores", str(scores_path),
           "--score-manifest", str(manifest), "--tm-matrix", str(tm_path),
           "--tm-manifest", str(tm_manifest)]
     main(command)
     params = json.loads(output.read_text())
     assert params["scheme"] == "target-aware-compbias20-raw-sw-empirical-v1"
-    assert params["audit_raw_evalue_ceiling"] == 120
+    assert params["validation_status"] == f"audited-e0-le-{ceiling}"
+    assert params["audit_raw_evalue_ceiling"] == ceiling
+    assert params["empirical_calibration"]["fit_raw_evalue_ceiling"] == ceiling
     assert params["empirical_calibration"]["upper_tail_rule"] == "anchored-power-cap"
     assert params["empirical_calibration"]["background_scaling"] == "complete-eligible-clusters"
     searcher = Searcher(str(db), matrix_path=str(matrix_path),