fix: semantic-type and domain-validator edge cases from adversarial QA pass

src/freshdata/domains/finance/validator.py

@@ -181,11 +181,15 @@ def _check_balanced(self, df: pd.DataFrame, mapping: ColumnMapping, rule: Rule)
         credit = _to_numeric(df[mapping.actual("credit")]).fillna(0.0)
         tolerance = float(rule.params.get("tolerance", 0.0))
         work = pd.DataFrame({"_txn": df[txn], "_d": debit, "_c": credit}, index=df.index)
+        # Only rows with an identified transaction can be flagged; dropping NaN
+        # keys up front also keeps groupby-transform from raising when every
+        # transaction id is missing (pandas chokes on zero groups).
+        work = work[work["_txn"].notna()]
+        if work.empty:
+            return []
         sums = work.groupby("_txn")[["_d", "_c"]].transform("sum")
         unbalanced = (sums["_d"] - sums["_c"]).abs() > tolerance
-        # Only flag rows that belong to an identified transaction.
-        unbalanced = unbalanced & df[txn].notna()
-        return df.index[unbalanced].tolist()
+        return df.index[unbalanced.reindex(df.index, fill_value=False)].tolist()
 
     def _check_both_sided(self, df: pd.DataFrame, mapping: ColumnMapping, rule: Rule) -> list[Any]:
         debit = _to_numeric(df[mapping.actual("debit")])

src/freshdata/fieldcheck.py

@@ -94,6 +94,13 @@ def _safe_fullmatch(pattern: str, value: str) -> bool:
 _NUMERIC_TYPES = frozenset(
     {"numeric", "integer", "float", "currency_amount", "rate", "percentage"})
 _DATE_TYPES = frozenset({"date", "datetime", "date_like"})
+#: Every semantic_type with a dedicated check. Anything else only validates
+#: through the spec's own pattern/allowed_values/reference/bounds.
+_KNOWN_SEMANTIC_TYPES = _NUMERIC_TYPES | _DATE_TYPES | frozenset({
+    "company_name", "entity_name", "person_name", "city", "country",
+    "free_text", "text", "identifier", "account_number", "ticker",
+    "stock_ticker", "email", "url", "phone",
+})
 
 
 def detect_value_type(value: Any) -> str:
@@ -743,7 +750,7 @@ def _outlier_issues(
 
 def validate_fields(
     df: pd.DataFrame,
-    schema: Mapping[str, FieldSpec] | None = None,
+    schema: Mapping[str, FieldSpec | str] | None = None,
     *,
     policy: RemediationPolicy | None = None,
     infer_unspecified: bool = True,
@@ -779,7 +786,37 @@ def validate_fields(
         when ``policy.normalize_text`` is true; every change is audited.
     """
     policy = policy or RemediationPolicy()
-    schema = dict(schema or {})
+    specs: dict[Any, FieldSpec] = {}
+    for col, raw_spec in dict(schema or {}).items():
+        spec = raw_spec
+        if isinstance(spec, str):  # shorthand: {"price": "numeric"}
+            spec = FieldSpec(semantic_type=spec)
+        elif not isinstance(spec, FieldSpec):
+            raise TypeError(
+                f"schema[{col!r}] must be a FieldSpec or a semantic-type string, "
+                f"got {type(spec).__name__}"
+            )
+        if (
+            spec.semantic_type is not None
+            and spec.semantic_type not in _KNOWN_SEMANTIC_TYPES
+            and spec.pattern is None
+            and spec.allowed_values is None
+            and spec.reference is None
+            and spec.min_value is None
+            and spec.max_value is None
+            and spec.max_length is None
+        ):
+            import warnings  # noqa: PLC0415
+
+            warnings.warn(
+                f"FieldSpec for {col!r} declares unknown semantic_type "
+                f"{spec.semantic_type!r} and no other constraint, so nothing "
+                f"will be validated; known types: {sorted(_KNOWN_SEMANTIC_TYPES)}",
+                UserWarning,
+                stacklevel=2,
+            )
+        specs[col] = spec
+    schema = specs
     report = FieldValidationReport(n_rows=len(df))
 
     checked_cols = [c for c in df.columns if c in schema] + (

src/freshdata/semantic/semantic_types.py

@@ -130,8 +130,13 @@ def _share(values: list[str], predicate) -> float:
     return sum(1 for v in values if predicate(v)) / len(values)
 
 
-def _content_detect(values: list[str]) -> tuple[str, float] | None:
-    """The strongest deterministic content signal over the distinct sample."""
+#: Strict-format types: a 0% content match over the distinct sample actively
+#: contradicts these, so a name hint alone can never certify them.
+_STRICT_FORMAT_TYPES = frozenset({"email", "url", "phone", "postal_code"})
+
+
+def _type_shares(values: list[str]) -> dict[str, float]:
+    """Per-type share of distinct values matching each content detector."""
     checks: tuple[tuple[str, float], ...] = (
         ("email", _share(values, lambda v: bool(_EMAIL_RE.match(v)))),
         ("url", _share(values, lambda v: bool(_URL_RE.match(v)))),
@@ -152,7 +157,12 @@ def _content_detect(values: list[str]) -> tuple[str, float] | None:
         ("person_name", _share(values, lambda v: bool(_PERSON_NAME_RE.match(v)))),
         ("category_code", _share(values, lambda v: bool(_CATEGORY_CODE_RE.match(v)))),
     )
-    best_type, best_share = max(checks, key=lambda pair: pair[1])
+    return dict(checks)
+
+
+def _content_detect(shares: dict[str, float]) -> tuple[str, float] | None:
+    """The strongest deterministic content signal over the distinct sample."""
+    best_type, best_share = max(shares.items(), key=lambda pair: pair[1])
     if best_share < 0.6:
         return None
     return best_type, best_share
@@ -198,6 +208,24 @@ def infer_semantic_type(
             "free_text", 0.8, (SemanticEvidence("column_role", "role inference: text", 0.0),)
         )
 
+    # Boolean columns inherently have ~2 distinct values, so they must be
+    # recognised before the distinct-support gate or they never would be.
+    # A bare {"0", "1"} column stays ungated: it is just as likely numeric.
+    if (
+        2 <= len(values) < MIN_DISTINCT_SUPPORT
+        and all(v.casefold() in _BOOL_VALUES for v in values)
+        and not {v.casefold() for v in values} <= {"0", "1"}
+    ):
+        return SemanticTypeResult(
+            "boolean_like",
+            0.9,
+            (
+                SemanticEvidence(
+                    "pattern", f"all {len(values)} distinct values are boolean tokens", 0.0
+                ),
+            ),
+        )
+
     if len(values) < MIN_DISTINCT_SUPPORT:
         return SemanticTypeResult(
             "unknown",
@@ -209,7 +237,8 @@ def infer_semantic_type(
             ),
         )
 
-    detected = _content_detect(values)
+    shares = _type_shares(values)
+    detected = _content_detect(shares)
     name_type = _name_hint(name)
 
     if detected is not None:
@@ -262,8 +291,21 @@ def infer_semantic_type(
             return SemanticTypeResult(vote_type, min(0.7, vote_score), tuple(evidence))
 
     if name_type is not None:
-        evidence.append(SemanticEvidence("pattern", f"column name suggests {name_type}", 0.0))
-        return SemanticTypeResult(name_type, 0.5, tuple(evidence))
+        # A strict-format name hint (email/url/phone/postal_code) that not one
+        # sampled value satisfies is contradicted by the data, not supported.
+        if name_type in _STRICT_FORMAT_TYPES and shares.get(name_type, 0.0) == 0.0:
+            evidence.append(
+                SemanticEvidence(
+                    "conflict",
+                    f"column name suggests {name_type} but no sampled value matches it",
+                    0.0,
+                )
+            )
+        else:
+            evidence.append(
+                SemanticEvidence("pattern", f"column name suggests {name_type}", 0.0)
+            )
+            return SemanticTypeResult(name_type, 0.5, tuple(evidence))
 
     if role == "categorical":
         return SemanticTypeResult(

tests/domains/test_finance.py

@@ -190,3 +190,22 @@ def test_no_domain_path_unchanged(good_finance):
     a = fd.clean(plain, verbose=False)
     b = fd.clean(plain.copy(), verbose=False)
     pd.testing.assert_frame_equal(a, b)
+
+
+def test_balance_check_survives_all_missing_transaction_ids(good_finance):
+    # Regression: an all-NaN transaction_id column made groupby-transform
+    # raise IndexError (zero groups); rows without an id are simply exempt.
+    df = good_finance.copy()
+    df["transaction_id"] = None
+    _, rep = fd.clean(df, domain="finance", return_report=True, verbose=False)
+    fin006 = next(f for f in rep.domain_findings if f["rule_id"] == "FIN-006")
+    assert fin006["n_violations"] == 0
+    # Partially missing ids: identified transactions are still checked
+    # (T1 rows 0-1 unbalanced by the credit edit; T2's remaining row 2
+    # unbalanced by losing row 3), but the id-less row itself is exempt.
+    df2 = good_finance.copy()
+    df2.loc[1, "credit"] = 999.0            # T1 unbalanced
+    df2.loc[3, "transaction_id"] = None     # exempt row; leaves T2 one-sided
+    _, rep2 = fd.clean(df2, domain="finance", return_report=True, verbose=False)
+    fin006 = next(f for f in rep2.domain_findings if f["rule_id"] == "FIN-006")
+    assert fin006["n_violations"] == 3

tests/test_fieldcheck.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import sqlite3
+import warnings
 
 import pandas as pd
 import pytest
@@ -661,3 +662,32 @@ def test_high_volume_batch_smoke():
     assert [i.row for i in bad] == [1234]
     result = apply_field_policy(df, report)
     assert len(result.accepted) == n - 1
+
+
+def test_schema_accepts_semantic_type_string_shorthand():
+    df = pd.DataFrame({"price": ["$100", "$200", "apple", "$400", "$500", "$600"]})
+    report = validate_fields(df, {"price": "numeric"})
+    [issue] = [i for i in report.issues if i.severity == "error"]
+    assert issue.row == 2 and issue.classification == "semantic_mismatch"
+
+
+def test_schema_rejects_non_fieldspec_values():
+    df = pd.DataFrame({"price": [1.0]})
+    with pytest.raises(TypeError, match="FieldSpec or a semantic-type string"):
+        validate_fields(df, {"price": 42})
+
+
+def test_unknown_semantic_type_without_constraints_warns():
+    # Regression: a typo'd semantic_type silently validated nothing.
+    df = pd.DataFrame({"price": ["apple"]})
+    with pytest.warns(UserWarning, match="unknown semantic_type 'martian'"):
+        validate_fields(df, {"price": FieldSpec(semantic_type="martian")})
+    # A spec with its own executable constraint stays silent (documented fallback):
+    # no *unknown-semantic-type* warning, though unrelated library warnings
+    # (e.g. pandas/numpy internals on older stacks) are not our concern here.
+    with warnings.catch_warnings(record=True) as caught:
+        warnings.simplefilter("always")
+        report = validate_fields(
+            df, {"price": FieldSpec(semantic_type="martian", pattern=r"\d+")})
+    assert not any("unknown semantic_type" in str(w.message) for w in caught)
+    assert any(i.rule == "pattern" for i in report.issues) or report.issues

tests/test_semantic_types.py

@@ -93,12 +93,39 @@ def test_role_signals_map_to_types():
 
 
 def test_name_hint_used_when_content_is_inconclusive():
-    values = [f"C-{i}-x{i}" for i in range(8)]  # matches no detector at 60%
+    # 3/8 emails: below the 60% detection bar but not contradicted, so the
+    # column name may still suggest the type at hint-level confidence.
+    values = EMAILS[:3] + [f"C-{i}-x{i}" for i in range(5)]
     result = infer_semantic_type("email_addr", _series(values))
     assert result.semantic_type == "email"
     assert result.confidence == pytest.approx(0.5)
 
 
+def test_strict_name_hint_vetoed_when_content_contradicts():
+    # A column *named* email whose sampled values match email 0% must not be
+    # certified as email by the name alone (regression: name-hint false positive).
+    result = infer_semantic_type("email", _series([str(i) for i in range(1, 9)]))
+    assert result.semantic_type != "email"
+    assert any(e.kind == "conflict" for e in result.evidence)
+
+
+def test_boolean_detected_below_distinct_support():
+    # Booleans inherently have ~2 distinct values; they must not be swallowed
+    # by the distinct-support gate (regression: boolean_like was unreachable).
+    two_token = infer_semantic_type("subscribed", _series(["yes", "no", "yes", "no"]))
+    assert two_token.semantic_type == "boolean_like"
+    assert two_token.confidence >= 0.8
+
+    bool_dtype = infer_semantic_type("active", pd.Series([True, False] * 5))
+    assert bool_dtype.semantic_type == "boolean_like"
+
+
+def test_bare_binary_01_not_forced_boolean():
+    # {"0", "1"} is just as likely a numeric indicator: stays gated.
+    result = infer_semantic_type("flag", _series(["0", "1", "0", "1"]))
+    assert result.semantic_type == "unknown"
+
+
 def test_infer_roles_gains_additive_columns():
     df = pd.DataFrame(
         {