Warn about group labels differing only in case or spacing

assets/profiler-engine.js

@@ -1628,7 +1628,15 @@
     if (maxAge === undefined) maxAge = MAX_AGE;
     var flags = [];
     dimensions.forEach(function (d) {
+      var foldedLabels = Object.create(null);
       d.groups.forEach(function (g) {
+        var folded = foldKeyCase(g.label.replace(KEY_TRIM, ''));
+        if (Object.prototype.hasOwnProperty.call(foldedLabels, folded)) {
+          flags.push(d.name + ": '" + foldedLabels[folded] + "' and '" + g.label +
+                     "' differ only in case or spacing");
+        } else {
+          foldedLabels[folded] = g.label;
+        }
         if (d.under_represented.indexOf(g.label) !== -1) {
           flags.push(d.name + ": '" + g.label + "' is under-represented (" +
                      (g.share * 100).toFixed(1) + '%)');

faircode/SPEC.md

@@ -127,6 +127,16 @@ Category labels are literal data, including names such as `__proto__`, `construc
 and `toString`. They must be retained in distinct counts, groups, intersections,
 reference baselines, and drift comparisons, with no inherited-property lookups.
 
+**Label collisions (informational).** Within each dimension, compare distinct observed
+group labels after trimming the Unicode White_Space characters plus U+FEFF and
+applying full Unicode case-folding. If labels
+match, add a flag such as `sex: 'M' and 'm' differ only in case or spacing`.
+Compare each subsequent matching label to the first in the groups' count-descending,
+label-ascending order, once per label rather than per row. Keep the original labels,
+groups, counts, shares, metrics, and scores unchanged. Do not apply join keys' NFKC
+or numeric normalization: `01` and `1`, or `Ａ` and `A`, remain distinct without this
+warning. The flags appear in the terminal, browser, and exported reports.
+
 - **shares** - the `p_i`, descending, with raw counts.
 - **ci_low / ci_high** - a 95% **Wilson score interval** on each group's share, so a share read off a small sample carries its sampling uncertainty. For a group with count `c` out of `N_nonnull = n`, `p = c/n`, `z = 1.959963984540054`:
   - `center = (p + z²/2n) / (1 + z²/n)`, `margin = (z / (1 + z²/n)) · √( p(1−p)/n + z²/4n² )`
@@ -232,7 +242,7 @@ dict, JS uses a plain object):
 `flags` is a human-readable list assembled from: every `under_represented` group, every dimension
 with `imbalance_ratio ≥ 3`, every dimension with `missing_pct ≥ 0.05`, every age dimension with
 `implausible_values` (an optional integer key present only when above zero, #840), and every
-intersectional gap.
+intersectional gap, plus the label-collision warnings from section 3.
 
 ---
 

faircode/profiler.py

@@ -27,6 +27,9 @@
 MAX_AGE = 120  # numeric ages above this are reported as implausible, not banded (SPEC 2)
 BIRTH_YEAR_MIN = 1900  # with age_reference_year, whole numbers from here up to it are birth years
 
+# Shared with proxy join keys; matches KEY_TRIM in profiler-engine.js.
+KEY_TRIM_CHARS = "\t\n\v\f\r \x85\xa0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff"
+
 # ASCII digits only, like the JS engine (#836): Python's `\d` also matches
 # Arabic-Indic/fullwidth digits, which the browser regex does not.
 _DATE_RE = re.compile(r"[0-9]{1,4}[/-][0-9]{1,2}[/-][0-9]{1,4}")
@@ -442,7 +445,16 @@ def _build_flags(dimensions: list[dict], intersections: list[dict],
                  max_age=MAX_AGE) -> list[str]:
     flags: list[str] = []
     for d in dimensions:
+        folded_labels = {}
         for g in d["groups"]:
+            folded = g["label"].strip(KEY_TRIM_CHARS).casefold()
+            if folded in folded_labels:
+                flags.append(
+                    f"{d['name']}: '{folded_labels[folded]}' and '{g['label']}' "
+                    "differ only in case or spacing"
+                )
+            else:
+                folded_labels[folded] = g["label"]
             if g["label"] in d["under_represented"]:
                 flags.append(
                     f"{d['name']}: '{g['label']}' is under-represented "

faircode/proxy.py

@@ -21,7 +21,7 @@
 
 import pandas as pd
 
-from .profiler import MAX_AGE, _age_band, _age_numbers, _is_categorical_age_sentinel, _looks_like_dates
+from .profiler import KEY_TRIM_CHARS, MAX_AGE, _age_band, _age_numbers, _is_categorical_age_sentinel, _looks_like_dates
 
 PROXY_ALPHA = 0.05
 
@@ -279,12 +279,6 @@ def key_columns(key, columns):
 _WHOLE_FLOAT = re.compile(r"([0-9]+)\.0+")
 
 
-# The characters trimmed from both ends of a key: the Unicode White_Space set plus U+FEFF
-# (a BOM that survived in a first cell), spelled out so Python's str.strip() and JS's
-# String.prototype.trim() cannot drift apart (#897). The same list is in profiler-engine.js.
-KEY_TRIM_CHARS = "\t\n\v\f\r \x85\xa0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff"
-
-
 def normalize_key_text(value):
     """Opt-in key normalisation (#859, #884): trim, apply Unicode compatibility composition (NFKC, so full-width `４２`/`Ａ１` equal `42`/`a1`, #935),
     case-fold (so `ß` equals `ss`), drop a whole-number fraction (`42.0`), and drop leading zeros from an all-digit

tests/test_js_parity_label_collisions.py

@@ -0,0 +1,96 @@
+"""Warnings for literal group labels that differ only in case or spacing."""
+
+import html
+import json
+import subprocess
+
+import pandas as pd
+import pytest
+
+from faircode import profile
+from faircode.report import to_csv, to_html, to_terminal
+from js_helpers import REPO_ROOT, _run_ui_exports
+
+
+def _profile_both(df):
+    script = (
+        "require(process.argv[1]);var E=globalThis.FairCodeProfiler;"
+        "process.stdout.write(JSON.stringify(E.profile(JSON.parse(process.argv[2]))));"
+    )
+    completed = subprocess.run(
+        ["node", "-e", script, str(REPO_ROOT / "assets" / "profiler-engine.js"),
+         json.dumps({"columns": list(df.columns), "rows": json.loads(df.to_json(orient="records"))})],
+        capture_output=True, text=True, encoding="utf-8", check=True,
+    )
+    result = profile(df)
+    assert json.loads(completed.stdout) == result
+    return result
+
+
+@pytest.mark.parametrize("column", ["sex", "race", "region", "segment", "age"])
+@pytest.mark.parametrize("labels", [
+    ("M", "m"),
+    ("m", " m "),
+    ("M", "\t m\u00a0"),
+    ("M", "\u0085m\u3000"),
+    ("M", "\ufeffm\ufeff"),
+    ("Stra\u00dfe", "STRASSE"),
+    ("\u03c3", "\u03c2"),
+    ("\ufb03", "FFI"),
+    ("__proto__", "__PROTO__"),
+    ("constructor", "CONSTRUCTOR"),
+    ("toString", "TOSTRING"),
+])
+def test_collisions_warn_without_merging_groups(column, labels):
+    first, second = labels
+    result = _profile_both(pd.DataFrame({column: [first] * 3 + [second] * 2 + [None]}))
+    warning = f"{column}: '{first}' and '{second}' differ only in case or spacing"
+    assert result["flags"].count(warning) == 1
+    dim = result["dimensions"][0]
+    assert dim["n_groups"] == 2
+    assert [(g["label"], g["count"], g["share"], g["small_group"]) for g in dim["groups"]] == [
+        (first, 3, 0.6, True), (second, 2, 0.4, True),
+    ]
+    assert dim["missing_pct"] == pytest.approx(1 / 6, abs=0.0001)
+    assert dim["imbalance_ratio"] == 1.5
+    assert dim["dimension_score"] == result["overall_score"] == 97
+
+
+@pytest.mark.parametrize("labels", [
+    ["M", "F"],
+    ["M", "M"],
+    ["M", "M", None],
+    [None, None],
+    ["01", "1", "1.0"],
+    ["\uff21", "A"],
+    ["e", "\u00e9"],
+    ["a b", "ab"],
+])
+def test_distinct_or_identical_labels_do_not_warn(labels):
+    result = _profile_both(pd.DataFrame({"sex": labels}))
+    assert not any("differ only in case or spacing" in f for f in result["flags"])
+
+
+def test_collision_warnings_are_per_dimension_and_not_repeated_per_row():
+    result = _profile_both(pd.DataFrame({
+        "sex": ["M"] * 30 + ["m"] * 20 + [" m "] * 10,
+        "race": ["M"] * 60,
+    }))
+    assert [f for f in result["flags"] if "differ only in case or spacing" in f] == [
+        "sex: 'M' and 'm' differ only in case or spacing",
+        "sex: 'M' and ' m ' differ only in case or spacing",
+    ]
+
+
+def test_collision_warning_reaches_reports_in_both_engines(tmp_path):
+    df = pd.DataFrame({"sex": ["M"] * 15 + ["m"] * 15})
+    path = tmp_path / "labels.csv"
+    df.to_csv(path, index=False)
+    result = _profile_both(df)
+    warning = "sex: 'M' and 'm' differ only in case or spacing"
+    assert warning in to_terminal(result)
+    assert warning in html.unescape(to_html(result))
+    assert warning in to_csv(result)
+    web = _run_ui_exports(path, False)
+    assert warning in html.unescape(web["html"])
+    assert web["csv"].replace("\r\n", "\n") == to_csv(result).replace("\r\n", "\n")