diff --git a/assets/profiler-engine.js b/assets/profiler-engine.js index 7b28f0d..ac4b070 100644 --- a/assets/profiler-engine.js +++ b/assets/profiler-engine.js @@ -1628,7 +1628,15 @@ if (maxAge === undefined) maxAge = MAX_AGE; var flags = []; dimensions.forEach(function (d) { + var foldedLabels = Object.create(null); d.groups.forEach(function (g) { + var folded = foldKeyCase(g.label.replace(KEY_TRIM, '')); + if (Object.prototype.hasOwnProperty.call(foldedLabels, folded)) { + flags.push(d.name + ": '" + foldedLabels[folded] + "' and '" + g.label + + "' differ only in case or spacing"); + } else { + foldedLabels[folded] = g.label; + } if (d.under_represented.indexOf(g.label) !== -1) { flags.push(d.name + ": '" + g.label + "' is under-represented (" + (g.share * 100).toFixed(1) + '%)'); diff --git a/faircode/SPEC.md b/faircode/SPEC.md index 9b9c0d7..49346bf 100644 --- a/faircode/SPEC.md +++ b/faircode/SPEC.md @@ -127,6 +127,16 @@ Category labels are literal data, including names such as `__proto__`, `construc and `toString`. They must be retained in distinct counts, groups, intersections, reference baselines, and drift comparisons, with no inherited-property lookups. +**Label collisions (informational).** Within each dimension, compare distinct observed +group labels after trimming the Unicode White_Space characters plus U+FEFF and +applying full Unicode case-folding. If labels +match, add a flag such as `sex: 'M' and 'm' differ only in case or spacing`. +Compare each subsequent matching label to the first in the groups' count-descending, +label-ascending order, once per label rather than per row. Keep the original labels, +groups, counts, shares, metrics, and scores unchanged. Do not apply join keys' NFKC +or numeric normalization: `01` and `1`, or `Ａ` and `A`, remain distinct without this +warning. The flags appear in the terminal, browser, and exported reports. + - **shares** - the `p_i`, descending, with raw counts. - **ci_low / ci_high** - a 95% **Wilson score interval** on each group's share, so a share read off a small sample carries its sampling uncertainty. For a group with count `c` out of `N_nonnull = n`, `p = c/n`, `z = 1.959963984540054`: - `center = (p + z²/2n) / (1 + z²/n)`, `margin = (z / (1 + z²/n)) · √( p(1−p)/n + z²/4n² )` @@ -232,7 +242,7 @@ dict, JS uses a plain object): `flags` is a human-readable list assembled from: every `under_represented` group, every dimension with `imbalance_ratio ≥ 3`, every dimension with `missing_pct ≥ 0.05`, every age dimension with `implausible_values` (an optional integer key present only when above zero, #840), and every -intersectional gap. +intersectional gap, plus the label-collision warnings from section 3. --- diff --git a/faircode/profiler.py b/faircode/profiler.py index 35391b9..f80f54d 100644 --- a/faircode/profiler.py +++ b/faircode/profiler.py @@ -27,6 +27,9 @@ MAX_AGE = 120 # numeric ages above this are reported as implausible, not banded (SPEC 2) BIRTH_YEAR_MIN = 1900 # with age_reference_year, whole numbers from here up to it are birth years +# Shared with proxy join keys; matches KEY_TRIM in profiler-engine.js. +KEY_TRIM_CHARS = "\t\n\v\f\r \x85\xa0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff" + # ASCII digits only, like the JS engine (#836): Python's `\d` also matches # Arabic-Indic/fullwidth digits, which the browser regex does not. _DATE_RE = re.compile(r"[0-9]{1,4}[/-][0-9]{1,2}[/-][0-9]{1,4}") @@ -442,7 +445,16 @@ def _build_flags(dimensions: list[dict], intersections: list[dict], max_age=MAX_AGE) -> list[str]: flags: list[str] = [] for d in dimensions: + folded_labels = {} for g in d["groups"]: + folded = g["label"].strip(KEY_TRIM_CHARS).casefold() + if folded in folded_labels: + flags.append( + f"{d['name']}: '{folded_labels[folded]}' and '{g['label']}' " + "differ only in case or spacing" + ) + else: + folded_labels[folded] = g["label"] if g["label"] in d["under_represented"]: flags.append( f"{d['name']}: '{g['label']}' is under-represented " diff --git a/faircode/proxy.py b/faircode/proxy.py index a4624fe..cb94559 100644 --- a/faircode/proxy.py +++ b/faircode/proxy.py @@ -21,7 +21,7 @@ import pandas as pd -from .profiler import MAX_AGE, _age_band, _age_numbers, _is_categorical_age_sentinel, _looks_like_dates +from .profiler import KEY_TRIM_CHARS, MAX_AGE, _age_band, _age_numbers, _is_categorical_age_sentinel, _looks_like_dates PROXY_ALPHA = 0.05 @@ -279,12 +279,6 @@ def key_columns(key, columns): _WHOLE_FLOAT = re.compile(r"([0-9]+)\.0+") -# The characters trimmed from both ends of a key: the Unicode White_Space set plus U+FEFF -# (a BOM that survived in a first cell), spelled out so Python's str.strip() and JS's -# String.prototype.trim() cannot drift apart (#897). The same list is in profiler-engine.js. -KEY_TRIM_CHARS = "\t\n\v\f\r \x85\xa0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff" - - def normalize_key_text(value): """Opt-in key normalisation (#859, #884): trim, apply Unicode compatibility composition (NFKC, so full-width `４２`/`Ａ１` equal `42`/`a1`, #935), case-fold (so `ß` equals `ss`), drop a whole-number fraction (`42.0`), and drop leading zeros from an all-digit diff --git a/tests/test_js_parity_label_collisions.py b/tests/test_js_parity_label_collisions.py new file mode 100644 index 0000000..edb7702 --- /dev/null +++ b/tests/test_js_parity_label_collisions.py @@ -0,0 +1,96 @@ +"""Warnings for literal group labels that differ only in case or spacing.""" + +import html +import json +import subprocess + +import pandas as pd +import pytest + +from faircode import profile +from faircode.report import to_csv, to_html, to_terminal +from js_helpers import REPO_ROOT, _run_ui_exports + + +def _profile_both(df): + script = ( + "require(process.argv[1]);var E=globalThis.FairCodeProfiler;" + "process.stdout.write(JSON.stringify(E.profile(JSON.parse(process.argv[2]))));" + ) + completed = subprocess.run( + ["node", "-e", script, str(REPO_ROOT / "assets" / "profiler-engine.js"), + json.dumps({"columns": list(df.columns), "rows": json.loads(df.to_json(orient="records"))})], + capture_output=True, text=True, encoding="utf-8", check=True, + ) + result = profile(df) + assert json.loads(completed.stdout) == result + return result + + +@pytest.mark.parametrize("column", ["sex", "race", "region", "segment", "age"]) +@pytest.mark.parametrize("labels", [ + ("M", "m"), + ("m", " m "), + ("M", "\t m\u00a0"), + ("M", "\u0085m\u3000"), + ("M", "\ufeffm\ufeff"), + ("Stra\u00dfe", "STRASSE"), + ("\u03c3", "\u03c2"), + ("\ufb03", "FFI"), + ("__proto__", "__PROTO__"), + ("constructor", "CONSTRUCTOR"), + ("toString", "TOSTRING"), +]) +def test_collisions_warn_without_merging_groups(column, labels): + first, second = labels + result = _profile_both(pd.DataFrame({column: [first] * 3 + [second] * 2 + [None]})) + warning = f"{column}: '{first}' and '{second}' differ only in case or spacing" + assert result["flags"].count(warning) == 1 + dim = result["dimensions"][0] + assert dim["n_groups"] == 2 + assert [(g["label"], g["count"], g["share"], g["small_group"]) for g in dim["groups"]] == [ + (first, 3, 0.6, True), (second, 2, 0.4, True), + ] + assert dim["missing_pct"] == pytest.approx(1 / 6, abs=0.0001) + assert dim["imbalance_ratio"] == 1.5 + assert dim["dimension_score"] == result["overall_score"] == 97 + + +@pytest.mark.parametrize("labels", [ + ["M", "F"], + ["M", "M"], + ["M", "M", None], + [None, None], + ["01", "1", "1.0"], + ["\uff21", "A"], + ["e", "\u00e9"], + ["a b", "ab"], +]) +def test_distinct_or_identical_labels_do_not_warn(labels): + result = _profile_both(pd.DataFrame({"sex": labels})) + assert not any("differ only in case or spacing" in f for f in result["flags"]) + + +def test_collision_warnings_are_per_dimension_and_not_repeated_per_row(): + result = _profile_both(pd.DataFrame({ + "sex": ["M"] * 30 + ["m"] * 20 + [" m "] * 10, + "race": ["M"] * 60, + })) + assert [f for f in result["flags"] if "differ only in case or spacing" in f] == [ + "sex: 'M' and 'm' differ only in case or spacing", + "sex: 'M' and ' m ' differ only in case or spacing", + ] + + +def test_collision_warning_reaches_reports_in_both_engines(tmp_path): + df = pd.DataFrame({"sex": ["M"] * 15 + ["m"] * 15}) + path = tmp_path / "labels.csv" + df.to_csv(path, index=False) + result = _profile_both(df) + warning = "sex: 'M' and 'm' differ only in case or spacing" + assert warning in to_terminal(result) + assert warning in html.unescape(to_html(result)) + assert warning in to_csv(result) + web = _run_ui_exports(path, False) + assert warning in html.unescape(web["html"]) + assert web["csv"].replace("\r\n", "\n") == to_csv(result).replace("\r\n", "\n")