From c91a6c7bba87aa5b16338e25c8b9520d4f4fd723 Mon Sep 17 00:00:00 2001 From: Hongxiang Li Date: Thu, 5 Nov 2026 23:34:23 +0800 Subject: [PATCH] Fix empirical E-value audit status for configurable ceilings --- rfoldsearch/experimental_calibrate.py | 2 +- rfoldsearch/experimental_evalue.py | 34 +++++++++++- rfoldsearch/predicted_evalue.py | 2 +- tests/test_empirical_audit_metadata.py | 58 +++++++++++++++++++++ tests/test_experimental_empirical_evalue.py | 26 +++++++-- tests/test_predicted_empirical_evalue.py | 26 +++++++-- 6 files changed, 135 insertions(+), 13 deletions(-) create mode 100644 tests/test_empirical_audit_metadata.py diff --git a/rfoldsearch/experimental_calibrate.py b/rfoldsearch/experimental_calibrate.py index 6fba9e8..d9f613c 100644 --- a/rfoldsearch/experimental_calibrate.py +++ b/rfoldsearch/experimental_calibrate.py @@ -305,7 +305,7 @@ def fit_files(args, *, predicted=False): "n_chains": len(ids), "n_clusters": audit["n_clusters"], "target_database_fingerprint": target_digest, "calibration_context_fingerprint": context_digest, - "validation_status": "audited-e0-le-100", + "validation_status": f"audited-e0-le-{audit_ceiling:.17g}", "audit_raw_evalue_ceiling": audit_ceiling, "target_aware_gumbel": model, "empirical_calibration": calibration, diff --git a/rfoldsearch/experimental_evalue.py b/rfoldsearch/experimental_evalue.py index 4866039..7505acb 100644 --- a/rfoldsearch/experimental_evalue.py +++ b/rfoldsearch/experimental_evalue.py @@ -287,13 +287,42 @@ def apply_empirical(evalues, calibration): return float(y) if np.ndim(y) == 0 else y +def validate_audit_metadata(params): + """Check the audited range, including files with the old fixed-100 label.""" + status = params.get("validation_status", "") + prefix = "audited-e0-le-" + if not isinstance(status, str) or not status.startswith(prefix): + raise ValueError("empirical E-values lack a completed structural audit") + try: + labelled = float(status[len(prefix):]) + ceiling = float(params.get("audit_raw_evalue_ceiling", labelled)) + except (TypeError, ValueError): + raise ValueError("invalid empirical audit ceiling") from None + if any(not math.isfinite(value) or value < 100 for value in (labelled, ceiling)): + raise ValueError("empirical audit ceilings must be finite and at least 100") + # Older fitters always wrote 100 in the label, even when the explicit + # audit_raw_evalue_ceiling recorded a larger, completed audit. + if labelled != ceiling and labelled != 100: + raise ValueError("empirical audit label and ceiling disagree") + calibration = params["empirical_calibration"] + try: + fit_ceiling = float(calibration.get("fit_raw_evalue_ceiling", ceiling)) + last_knot = float(calibration["raw_evalues"][-1]) + fit_max = float(calibration.get("fit_raw_evalue_max", last_knot)) + except (TypeError, ValueError): + raise ValueError("invalid empirical calibration range") from None + if (not math.isfinite(fit_ceiling) or not 0 < fit_ceiling <= ceiling + or any(not math.isfinite(value) or not 0 < value <= fit_ceiling + for value in (last_knot, fit_max))): + raise ValueError("empirical calibration range exceeds its audit or fitting ceiling") + + def validate_parameters(params, n_clusters, target_fingerprint, context_fingerprint): if params.get("scheme") != SCHEME or params.get("score_definition") != "raw-sw": raise ValueError("unsupported experimental E-value scheme or score") if (params.get("target_database_fingerprint") != target_fingerprint or params.get("calibration_context_fingerprint") != context_fingerprint - or int(params.get("n_clusters", -1)) != n_clusters - or params.get("validation_status") != "audited-e0-le-100"): + or int(params.get("n_clusters", -1)) != n_clusters): raise ValueError("experimental E-values belong to another database or lack audit") model = params["target_aware_gumbel"] _, _, _, _, multiplicities = _model_arrays(model) @@ -303,3 +332,4 @@ def validate_parameters(params, n_clusters, target_fingerprint, context_fingerpr if int(calibration.get("n_clusters", -1)) != n_clusters: raise ValueError("empirical map cluster count does not match database") apply_empirical(np.array([0.0, 1.0]), calibration) + validate_audit_metadata(params) diff --git a/rfoldsearch/predicted_evalue.py b/rfoldsearch/predicted_evalue.py index ec94d0b..4c00de8 100644 --- a/rfoldsearch/predicted_evalue.py +++ b/rfoldsearch/predicted_evalue.py @@ -13,7 +13,6 @@ def validate_parameters(params, n_clusters, target_fingerprint, if (params.get("scheme") != SCHEME or params.get("score_definition") != "compbias20-raw-sw" or params.get("composition_window") != compbias.WINDOW - or params.get("validation_status") != "audited-e0-le-100" or params.get("target_database_fingerprint") != target_fingerprint or params.get("calibration_context_fingerprint") != context_fingerprint or int(params.get("n_clusters", -1)) != n_clusters): @@ -27,6 +26,7 @@ def validate_parameters(params, n_clusters, target_fingerprint, or calibration.get("background_scaling") != "complete-eligible-clusters"): raise ValueError("predicted empirical calibration needs the full eligible background") core.apply_empirical(np.array([0.0, 1.0]), calibration) + core.validate_audit_metadata(params) fitted = np.asarray(params["composition_background"], dtype=float) if fitted.ndim != 1 or not len(fitted): raise ValueError("invalid predicted composition background") diff --git a/tests/test_empirical_audit_metadata.py b/tests/test_empirical_audit_metadata.py new file mode 100644 index 0000000..34aed35 --- /dev/null +++ b/tests/test_empirical_audit_metadata.py @@ -0,0 +1,58 @@ +"""Reject unaudited ranges without invalidating historical parameter files.""" +import pytest + +from rfoldsearch.experimental_evalue import validate_audit_metadata + + +@pytest.mark.parametrize("status,ceiling,fit_ceiling,fit_max,knots", [ + ("audited-e0-le-100", None, 100, 99, [.1, 99]), + ("audited-e0-le-400", 400, 400, 399, [.1, 399]), + ("audited-e0-le-100", 400, 400, 399, [.1, 399]), + ("audited-e0-le-400.125", 400.125, 400.125, 400.1, [.1, 400.1]), + ("audited-e0-le-400", 400, 100, 99, [.1, 99]), +]) +def test_supported_audit_ranges(status, ceiling, fit_ceiling, fit_max, knots): + params = {"validation_status": status, "empirical_calibration": { + "fit_raw_evalue_ceiling": fit_ceiling, "fit_raw_evalue_max": fit_max, + "raw_evalues": knots}} + if ceiling is not None: + params["audit_raw_evalue_ceiling"] = ceiling + validate_audit_metadata(params) + + +@pytest.mark.parametrize("changes", [ + {"validation_status": "structural-audit-not-recorded"}, + {"validation_status": "audited-e0-le-nan"}, + {"validation_status": "audited-e0-le-400junk"}, + {"validation_status": None}, + {"validation_status": "audited-e0-le-500"}, + {"audit_raw_evalue_ceiling": 100}, + {"audit_raw_evalue_ceiling": 500}, + {"audit_raw_evalue_ceiling": float("inf")}, + {"audit_raw_evalue_ceiling": 99}, + {"audit_raw_evalue_ceiling": None}, +]) +def test_invalid_audit_ranges(changes): + params = {"validation_status": "audited-e0-le-400", + "audit_raw_evalue_ceiling": 400, + "empirical_calibration": {"fit_raw_evalue_ceiling": 400, + "fit_raw_evalue_max": 399, "raw_evalues": [.1, 399]}} + params.update(changes) + with pytest.raises(ValueError): + validate_audit_metadata(params) + + +@pytest.mark.parametrize("changes", [ + {"fit_raw_evalue_ceiling": 500}, + {"fit_raw_evalue_ceiling": float("nan")}, + {"fit_raw_evalue_max": 401}, + {"fit_raw_evalue_max": float("inf")}, + {"raw_evalues": [.1, 401]}, +]) +def test_calibration_cannot_exceed_audited_range(changes): + calibration = {"fit_raw_evalue_ceiling": 400, + "fit_raw_evalue_max": 399, "raw_evalues": [.1, 399]} + calibration.update(changes) + with pytest.raises(ValueError, match="calibration range"): + validate_audit_metadata({"validation_status": "audited-e0-le-400", + "audit_raw_evalue_ceiling": 400, "empirical_calibration": calibration}) diff --git a/tests/test_experimental_empirical_evalue.py b/tests/test_experimental_empirical_evalue.py index 23a1a7c..22bd411 100644 --- a/tests/test_experimental_empirical_evalue.py +++ b/tests/test_experimental_empirical_evalue.py @@ -90,7 +90,8 @@ def test_fitter_uses_clean_background_for_both_stages(): assert uncensored["censoring"] == "none" -def test_experimental_cli_fit_attach_and_search(tmp_path): +@pytest.mark.parametrize("ceiling", [100, 400]) +def test_experimental_cli_fit_attach_and_search(tmp_path, ceiling): rng = np.random.default_rng(41) entries = add_coords([{"id": f"t{i}", "tokens": rng.integers(0, 20, 28 + i % 7).tolist(), "cluster": f"c{i}"} for i in range(32)]) @@ -122,8 +123,12 @@ def test_experimental_cli_fit_attach_and_search(tmp_path): main(["fit-experimental-evalues", "--db", str(db), "--labels", str(labels), "--workdir", str(tmp_path / "fit"), "--output", str(params_path), "--scores", str(score_path), "--score-manifest", str(score_manifest), - "--tm-matrix", str(tm_path), "--tm-manifest", str(tm_manifest)]) + "--tm-matrix", str(tm_path), "--tm-manifest", str(tm_manifest), + "--audit-ceiling", str(ceiling)]) params = json.loads(params_path.read_text()) + assert params["validation_status"] == f"audited-e0-le-{ceiling}" + assert params["audit_raw_evalue_ceiling"] == ceiling + assert params["empirical_calibration"]["fit_raw_evalue_ceiling"] == ceiling assert params["target_aware_gumbel"]["censoring"] == "left-censored-95th-percentile" assert params["empirical_calibration"]["upper_tail_rule"] == "anchored-power-cap" cluster_file = tmp_path / "clusters.tsv" @@ -140,7 +145,12 @@ def test_experimental_cli_fit_attach_and_search(tmp_path): hit["raw_evalue"], params["empirical_calibration"])) -def test_searcher_uses_matching_experimental_empirical_parameters(tmp_path): +@pytest.mark.parametrize("status,ceiling", [ + ("audited-e0-le-100", None), + ("audited-e0-le-400", 400), + ("audited-e0-le-100", 400), # Parameters from the old fixed-label fitter. +]) +def test_searcher_uses_matching_experimental_empirical_parameters(tmp_path, status, ceiling): entries = add_coords([{"id": f"t{i}", "tokens": [i % 5, (i + 1) % 5] * 15, "cluster": f"c{i}"} for i in range(5)]) target = database_fingerprint(entries, [e["cluster"] for e in entries], 80) @@ -150,7 +160,7 @@ def test_searcher_uses_matching_experimental_empirical_parameters(tmp_path): "scheme": EV.SCHEME, "score_definition": "raw-sw", "n_clusters": 5, "target_database_fingerprint": target, "calibration_context_fingerprint": context, - "validation_status": "audited-e0-le-100", + "validation_status": status, "target_aware_gumbel": { "coefficients_mu_logbeta": [[8., 0.], [0., 0.], [0., 0.], [0., 0.], [0., 0.]], @@ -163,6 +173,9 @@ def test_searcher_uses_matching_experimental_empirical_parameters(tmp_path): "low_tail_power_b": 1., "n_clusters": 5, }, } + if ceiling is not None: + params["audit_raw_evalue_ceiling"] = ceiling + params["empirical_calibration"]["fit_raw_evalue_ceiling"] = ceiling db = tmp_path / "db.pt" torch.save({"n_codes": 80, "entries": entries, "n_clusters": 5, "evalue_params": params}, db) @@ -174,6 +187,11 @@ def test_searcher_uses_matching_experimental_empirical_parameters(tmp_path): params["target_aware_gumbel"])) assert hit["evalue"] == pytest.approx(EV.apply_empirical( hit["raw_evalue"], params["empirical_calibration"])) + bad_audit = tmp_path / "bad_audit.json" + bad_audit.write_text(json.dumps(dict(params, + validation_status="audited-e0-le-400", audit_raw_evalue_ceiling=100))) + with pytest.raises(ValueError, match="audit label and ceiling disagree"): + Searcher(str(db), evalue_params_path=str(bad_audit)) params["target_database_fingerprint"] = "wrong" bad = tmp_path / "wrong.json" bad.write_text(json.dumps(params)) diff --git a/tests/test_predicted_empirical_evalue.py b/tests/test_predicted_empirical_evalue.py index b587ce4..a12f929 100644 --- a/tests/test_predicted_empirical_evalue.py +++ b/tests/test_predicted_empirical_evalue.py @@ -65,7 +65,12 @@ def test_compressed_gumbel_likelihood_matches_uncompressed_fit(): np.asarray(plain["coefficients_mu_logbeta"]), abs=1e-4) -def test_search_uses_compbias_only_for_evalue_and_rejects_wrong_context(tmp_path): +@pytest.mark.parametrize("status,ceiling", [ + ("audited-e0-le-100", None), + ("audited-e0-le-400", 400), + ("audited-e0-le-100", 400), # Parameters from the old fixed-label fitter. +]) +def test_search_uses_compbias_only_for_evalue_and_rejects_wrong_context(tmp_path, status, ceiling): entries = add_coords([{"id": f"t{i}", "tokens": [i % 5, (i + 1) % 5] * 15, "cluster": f"c{i}"} for i in range(5)]) tokens = [entry["tokens"] for entry in entries] @@ -79,7 +84,7 @@ def test_search_uses_compbias_only_for_evalue_and_rejects_wrong_context(tmp_path "composition_background": background.tolist(), "n_clusters": 5, "target_database_fingerprint": target, "calibration_context_fingerprint": context, - "validation_status": "audited-e0-le-100", + "validation_status": status, "target_aware_gumbel": { "coefficients_mu_logbeta": [[8., 0.], [0., 0.], [0., 0.], [0., 0.], [0., 0.]], @@ -93,6 +98,9 @@ def test_search_uses_compbias_only_for_evalue_and_rejects_wrong_context(tmp_path "background_scaling": "complete-eligible-clusters", }, } + if ceiling is not None: + params["audit_raw_evalue_ceiling"] = ceiling + params["empirical_calibration"]["fit_raw_evalue_ceiling"] = ceiling db = tmp_path / "db.pt" torch.save({"n_codes": 80, "entries": entries, "n_clusters": 5, "evalue_params": params}, db) @@ -111,6 +119,11 @@ def test_search_uses_compbias_only_for_evalue_and_rejects_wrong_context(tmp_path params["target_aware_gumbel"])) assert hit["evalue"] == pytest.approx(EV.apply_empirical( hit["raw_evalue"], params["empirical_calibration"])) + bad_audit = tmp_path / "bad_audit.json" + bad_audit.write_text(json.dumps(dict(params, + validation_status="audited-e0-le-400", audit_raw_evalue_ceiling=100))) + with pytest.raises(ValueError, match="audit label and ceiling disagree"): + Searcher(str(db), evalue_params_path=str(bad_audit)) params["composition_background"][0] += .01 bad = tmp_path / "wrong.json" bad.write_text(json.dumps(params)) @@ -118,7 +131,8 @@ def test_search_uses_compbias_only_for_evalue_and_rejects_wrong_context(tmp_path Searcher(str(db), evalue_params_path=str(bad)) -def test_fitting_cli_writes_attachable_predicted_parameters(tmp_path): +@pytest.mark.parametrize("ceiling", [100, 400]) +def test_fitting_cli_writes_attachable_predicted_parameters(tmp_path, ceiling): rng = np.random.default_rng(30) entries = add_coords([{"id": f"t{i}", "tokens": rng.integers(0, 15, 25 + i % 8).tolist(), "cluster": f"c{i}"} for i in range(40)]) @@ -163,14 +177,16 @@ def test_fitting_cli_writes_attachable_predicted_parameters(tmp_path): "--training-indices", str(training), "--calibration-indices", str(calibration), "--auxiliary-indices", str(auxiliary), "--workdir", str(tmp_path / "fit"), "--output", str(output), - "--audit-ceiling", "120", + "--audit-ceiling", str(ceiling), "--matrix", str(matrix_path), "--scores", str(scores_path), "--score-manifest", str(manifest), "--tm-matrix", str(tm_path), "--tm-manifest", str(tm_manifest)] main(command) params = json.loads(output.read_text()) assert params["scheme"] == "target-aware-compbias20-raw-sw-empirical-v1" - assert params["audit_raw_evalue_ceiling"] == 120 + assert params["validation_status"] == f"audited-e0-le-{ceiling}" + assert params["audit_raw_evalue_ceiling"] == ceiling + assert params["empirical_calibration"]["fit_raw_evalue_ceiling"] == ceiling assert params["empirical_calibration"]["upper_tail_rule"] == "anchored-power-cap" assert params["empirical_calibration"]["background_scaling"] == "complete-eligible-clusters" searcher = Searcher(str(db), matrix_path=str(matrix_path),