diff --git a/README.md b/README.md index 2a914d1..07018d3 100644 --- a/README.md +++ b/README.md @@ -1047,7 +1047,7 @@ provenance) instead of being tested; raise `--max-categorical-card` to test it a For CI, `--fail-under N` returns exit code `1` and explains the failing score on stderr when the overall representation score is below `N`; report output, including `--json`, remains on stdout. A score at or above the threshold exits -successfully. +successfully. `N` must be between 0 and 100; any other value exits `2` with an error. `--json` output carries a `provenance` block recording the SHA-256 of the input file, the `faircode` version, and the thresholds actually resolved for the run, diff --git a/faircode/SPEC.md b/faircode/SPEC.md index 115bcb3..2b71007 100644 --- a/faircode/SPEC.md +++ b/faircode/SPEC.md @@ -200,6 +200,11 @@ Grade bands: | D | 40–54 | Strong imbalance / sparse subgroups | | F | 0–39 | Severe imbalance or single-group dimensions | +`faircode profile --fail-under N` compares the score with `N` for a CI gate: exit 1 when the score is +below `N`, exit 0 otherwise. `N` must be between 0 and 100, the range of the score. Any other value, +including `nan`, exits 2 with "--fail-under must be between 0 and 100, got N" before the dataset is +read, because a value above 100 would fail every run and a negative one would never fail (#1037). + The score intentionally reflects **balance only**. Missing-data %, imbalance ratios, and intersectional gaps are surfaced as **flags** alongside the score, not folded into it - this keeps the two engines trivially in sync and the score easy to explain. diff --git a/faircode/cli.py b/faircode/cli.py index 5ba262d..80547a0 100644 --- a/faircode/cli.py +++ b/faircode/cli.py @@ -556,6 +556,11 @@ def main(argv: list[str] | None = None) -> int: if not args.sample and not args.csv: print("error: profile needs a csv argument (or --sample)", file=sys.stderr) return 2 + if args.fail_under is not None and not 0 <= args.fail_under <= 100: + # The score is 0-100: above that the gate always fails, below 0 it never does (#1037). + print(f"error: --fail-under must be between 0 and 100, got {args.fail_under:g}", + file=sys.stderr) + return 2 if args.proxy_alpha is not None and not 0 < args.proxy_alpha <= 1: print("error: --proxy-alpha must be in (0, 1]", file=sys.stderr) return 2 diff --git a/tests/test_cli.py b/tests/test_cli.py index ea220cd..aa83a5a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1730,3 +1730,22 @@ def test_reference_csv_with_a_blank_share_ignores_that_row(tmp_path, capsys): assert "NaN" not in out and "Infinity" not in out groups = {g["label"] for g in json.loads(out)["dimensions"][0]["reference"]["groups"]} assert groups == {"M", "F"} + + +def test_fail_under_outside_0_to_100_is_a_usage_error(tmp_path, capsys): + """#1037: 150 would fail every run and -5 would never fail, so both are rejected.""" + path = tmp_path / "balanced.csv" + path.write_text("sex\nM\nF\nM\nF\n", encoding="utf-8") + for value in ("150", "100.5", "-5", "-0.1", "nan", "inf"): + assert main(["profile", str(path), "--fail-under", value]) == 2 + captured = capsys.readouterr() + assert "--fail-under must be between 0 and 100, got " in captured.err + assert captured.out == "" + + +def test_fail_under_accepts_both_ends_of_the_range(tmp_path, capsys): + path = tmp_path / "balanced.csv" + path.write_text("sex\nM\nF\nM\nF\n", encoding="utf-8") + assert main(["profile", str(path), "--fail-under", "0"]) == 0 + assert main(["profile", str(path), "--fail-under", "100"]) == 0 + assert "must be between" not in capsys.readouterr().err