diff --git a/src/freshdata/domains/finance/validator.py b/src/freshdata/domains/finance/validator.py index bedd746..308e010 100644 --- a/src/freshdata/domains/finance/validator.py +++ b/src/freshdata/domains/finance/validator.py @@ -181,11 +181,15 @@ def _check_balanced(self, df: pd.DataFrame, mapping: ColumnMapping, rule: Rule) credit = _to_numeric(df[mapping.actual("credit")]).fillna(0.0) tolerance = float(rule.params.get("tolerance", 0.0)) work = pd.DataFrame({"_txn": df[txn], "_d": debit, "_c": credit}, index=df.index) + # Only rows with an identified transaction can be flagged; dropping NaN + # keys up front also keeps groupby-transform from raising when every + # transaction id is missing (pandas chokes on zero groups). + work = work[work["_txn"].notna()] + if work.empty: + return [] sums = work.groupby("_txn")[["_d", "_c"]].transform("sum") unbalanced = (sums["_d"] - sums["_c"]).abs() > tolerance - # Only flag rows that belong to an identified transaction. - unbalanced = unbalanced & df[txn].notna() - return df.index[unbalanced].tolist() + return df.index[unbalanced.reindex(df.index, fill_value=False)].tolist() def _check_both_sided(self, df: pd.DataFrame, mapping: ColumnMapping, rule: Rule) -> list[Any]: debit = _to_numeric(df[mapping.actual("debit")]) diff --git a/src/freshdata/fieldcheck.py b/src/freshdata/fieldcheck.py index ea3161c..f649b71 100644 --- a/src/freshdata/fieldcheck.py +++ b/src/freshdata/fieldcheck.py @@ -94,6 +94,13 @@ def _safe_fullmatch(pattern: str, value: str) -> bool: _NUMERIC_TYPES = frozenset( {"numeric", "integer", "float", "currency_amount", "rate", "percentage"}) _DATE_TYPES = frozenset({"date", "datetime", "date_like"}) +#: Every semantic_type with a dedicated check. Anything else only validates +#: through the spec's own pattern/allowed_values/reference/bounds. +_KNOWN_SEMANTIC_TYPES = _NUMERIC_TYPES | _DATE_TYPES | frozenset({ + "company_name", "entity_name", "person_name", "city", "country", + "free_text", "text", "identifier", "account_number", "ticker", + "stock_ticker", "email", "url", "phone", +}) def detect_value_type(value: Any) -> str: @@ -743,7 +750,7 @@ def _outlier_issues( def validate_fields( df: pd.DataFrame, - schema: Mapping[str, FieldSpec] | None = None, + schema: Mapping[str, FieldSpec | str] | None = None, *, policy: RemediationPolicy | None = None, infer_unspecified: bool = True, @@ -779,7 +786,37 @@ def validate_fields( when ``policy.normalize_text`` is true; every change is audited. """ policy = policy or RemediationPolicy() - schema = dict(schema or {}) + specs: dict[Any, FieldSpec] = {} + for col, raw_spec in dict(schema or {}).items(): + spec = raw_spec + if isinstance(spec, str): # shorthand: {"price": "numeric"} + spec = FieldSpec(semantic_type=spec) + elif not isinstance(spec, FieldSpec): + raise TypeError( + f"schema[{col!r}] must be a FieldSpec or a semantic-type string, " + f"got {type(spec).__name__}" + ) + if ( + spec.semantic_type is not None + and spec.semantic_type not in _KNOWN_SEMANTIC_TYPES + and spec.pattern is None + and spec.allowed_values is None + and spec.reference is None + and spec.min_value is None + and spec.max_value is None + and spec.max_length is None + ): + import warnings # noqa: PLC0415 + + warnings.warn( + f"FieldSpec for {col!r} declares unknown semantic_type " + f"{spec.semantic_type!r} and no other constraint, so nothing " + f"will be validated; known types: {sorted(_KNOWN_SEMANTIC_TYPES)}", + UserWarning, + stacklevel=2, + ) + specs[col] = spec + schema = specs report = FieldValidationReport(n_rows=len(df)) checked_cols = [c for c in df.columns if c in schema] + ( diff --git a/src/freshdata/semantic/semantic_types.py b/src/freshdata/semantic/semantic_types.py index 70baed6..be16b4c 100644 --- a/src/freshdata/semantic/semantic_types.py +++ b/src/freshdata/semantic/semantic_types.py @@ -130,8 +130,13 @@ def _share(values: list[str], predicate) -> float: return sum(1 for v in values if predicate(v)) / len(values) -def _content_detect(values: list[str]) -> tuple[str, float] | None: - """The strongest deterministic content signal over the distinct sample.""" +#: Strict-format types: a 0% content match over the distinct sample actively +#: contradicts these, so a name hint alone can never certify them. +_STRICT_FORMAT_TYPES = frozenset({"email", "url", "phone", "postal_code"}) + + +def _type_shares(values: list[str]) -> dict[str, float]: + """Per-type share of distinct values matching each content detector.""" checks: tuple[tuple[str, float], ...] = ( ("email", _share(values, lambda v: bool(_EMAIL_RE.match(v)))), ("url", _share(values, lambda v: bool(_URL_RE.match(v)))), @@ -152,7 +157,12 @@ def _content_detect(values: list[str]) -> tuple[str, float] | None: ("person_name", _share(values, lambda v: bool(_PERSON_NAME_RE.match(v)))), ("category_code", _share(values, lambda v: bool(_CATEGORY_CODE_RE.match(v)))), ) - best_type, best_share = max(checks, key=lambda pair: pair[1]) + return dict(checks) + + +def _content_detect(shares: dict[str, float]) -> tuple[str, float] | None: + """The strongest deterministic content signal over the distinct sample.""" + best_type, best_share = max(shares.items(), key=lambda pair: pair[1]) if best_share < 0.6: return None return best_type, best_share @@ -198,6 +208,24 @@ def infer_semantic_type( "free_text", 0.8, (SemanticEvidence("column_role", "role inference: text", 0.0),) ) + # Boolean columns inherently have ~2 distinct values, so they must be + # recognised before the distinct-support gate or they never would be. + # A bare {"0", "1"} column stays ungated: it is just as likely numeric. + if ( + 2 <= len(values) < MIN_DISTINCT_SUPPORT + and all(v.casefold() in _BOOL_VALUES for v in values) + and not {v.casefold() for v in values} <= {"0", "1"} + ): + return SemanticTypeResult( + "boolean_like", + 0.9, + ( + SemanticEvidence( + "pattern", f"all {len(values)} distinct values are boolean tokens", 0.0 + ), + ), + ) + if len(values) < MIN_DISTINCT_SUPPORT: return SemanticTypeResult( "unknown", @@ -209,7 +237,8 @@ def infer_semantic_type( ), ) - detected = _content_detect(values) + shares = _type_shares(values) + detected = _content_detect(shares) name_type = _name_hint(name) if detected is not None: @@ -262,8 +291,21 @@ def infer_semantic_type( return SemanticTypeResult(vote_type, min(0.7, vote_score), tuple(evidence)) if name_type is not None: - evidence.append(SemanticEvidence("pattern", f"column name suggests {name_type}", 0.0)) - return SemanticTypeResult(name_type, 0.5, tuple(evidence)) + # A strict-format name hint (email/url/phone/postal_code) that not one + # sampled value satisfies is contradicted by the data, not supported. + if name_type in _STRICT_FORMAT_TYPES and shares.get(name_type, 0.0) == 0.0: + evidence.append( + SemanticEvidence( + "conflict", + f"column name suggests {name_type} but no sampled value matches it", + 0.0, + ) + ) + else: + evidence.append( + SemanticEvidence("pattern", f"column name suggests {name_type}", 0.0) + ) + return SemanticTypeResult(name_type, 0.5, tuple(evidence)) if role == "categorical": return SemanticTypeResult( diff --git a/tests/domains/test_finance.py b/tests/domains/test_finance.py index b8c369b..0935c2f 100644 --- a/tests/domains/test_finance.py +++ b/tests/domains/test_finance.py @@ -190,3 +190,22 @@ def test_no_domain_path_unchanged(good_finance): a = fd.clean(plain, verbose=False) b = fd.clean(plain.copy(), verbose=False) pd.testing.assert_frame_equal(a, b) + + +def test_balance_check_survives_all_missing_transaction_ids(good_finance): + # Regression: an all-NaN transaction_id column made groupby-transform + # raise IndexError (zero groups); rows without an id are simply exempt. + df = good_finance.copy() + df["transaction_id"] = None + _, rep = fd.clean(df, domain="finance", return_report=True, verbose=False) + fin006 = next(f for f in rep.domain_findings if f["rule_id"] == "FIN-006") + assert fin006["n_violations"] == 0 + # Partially missing ids: identified transactions are still checked + # (T1 rows 0-1 unbalanced by the credit edit; T2's remaining row 2 + # unbalanced by losing row 3), but the id-less row itself is exempt. + df2 = good_finance.copy() + df2.loc[1, "credit"] = 999.0 # T1 unbalanced + df2.loc[3, "transaction_id"] = None # exempt row; leaves T2 one-sided + _, rep2 = fd.clean(df2, domain="finance", return_report=True, verbose=False) + fin006 = next(f for f in rep2.domain_findings if f["rule_id"] == "FIN-006") + assert fin006["n_violations"] == 3 diff --git a/tests/test_fieldcheck.py b/tests/test_fieldcheck.py index 274c582..24a67f5 100644 --- a/tests/test_fieldcheck.py +++ b/tests/test_fieldcheck.py @@ -3,6 +3,7 @@ from __future__ import annotations import sqlite3 +import warnings import pandas as pd import pytest @@ -661,3 +662,32 @@ def test_high_volume_batch_smoke(): assert [i.row for i in bad] == [1234] result = apply_field_policy(df, report) assert len(result.accepted) == n - 1 + + +def test_schema_accepts_semantic_type_string_shorthand(): + df = pd.DataFrame({"price": ["$100", "$200", "apple", "$400", "$500", "$600"]}) + report = validate_fields(df, {"price": "numeric"}) + [issue] = [i for i in report.issues if i.severity == "error"] + assert issue.row == 2 and issue.classification == "semantic_mismatch" + + +def test_schema_rejects_non_fieldspec_values(): + df = pd.DataFrame({"price": [1.0]}) + with pytest.raises(TypeError, match="FieldSpec or a semantic-type string"): + validate_fields(df, {"price": 42}) + + +def test_unknown_semantic_type_without_constraints_warns(): + # Regression: a typo'd semantic_type silently validated nothing. + df = pd.DataFrame({"price": ["apple"]}) + with pytest.warns(UserWarning, match="unknown semantic_type 'martian'"): + validate_fields(df, {"price": FieldSpec(semantic_type="martian")}) + # A spec with its own executable constraint stays silent (documented fallback): + # no *unknown-semantic-type* warning, though unrelated library warnings + # (e.g. pandas/numpy internals on older stacks) are not our concern here. + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + report = validate_fields( + df, {"price": FieldSpec(semantic_type="martian", pattern=r"\d+")}) + assert not any("unknown semantic_type" in str(w.message) for w in caught) + assert any(i.rule == "pattern" for i in report.issues) or report.issues diff --git a/tests/test_semantic_types.py b/tests/test_semantic_types.py index 2285167..9291338 100644 --- a/tests/test_semantic_types.py +++ b/tests/test_semantic_types.py @@ -93,12 +93,39 @@ def test_role_signals_map_to_types(): def test_name_hint_used_when_content_is_inconclusive(): - values = [f"C-{i}-x{i}" for i in range(8)] # matches no detector at 60% + # 3/8 emails: below the 60% detection bar but not contradicted, so the + # column name may still suggest the type at hint-level confidence. + values = EMAILS[:3] + [f"C-{i}-x{i}" for i in range(5)] result = infer_semantic_type("email_addr", _series(values)) assert result.semantic_type == "email" assert result.confidence == pytest.approx(0.5) +def test_strict_name_hint_vetoed_when_content_contradicts(): + # A column *named* email whose sampled values match email 0% must not be + # certified as email by the name alone (regression: name-hint false positive). + result = infer_semantic_type("email", _series([str(i) for i in range(1, 9)])) + assert result.semantic_type != "email" + assert any(e.kind == "conflict" for e in result.evidence) + + +def test_boolean_detected_below_distinct_support(): + # Booleans inherently have ~2 distinct values; they must not be swallowed + # by the distinct-support gate (regression: boolean_like was unreachable). + two_token = infer_semantic_type("subscribed", _series(["yes", "no", "yes", "no"])) + assert two_token.semantic_type == "boolean_like" + assert two_token.confidence >= 0.8 + + bool_dtype = infer_semantic_type("active", pd.Series([True, False] * 5)) + assert bool_dtype.semantic_type == "boolean_like" + + +def test_bare_binary_01_not_forced_boolean(): + # {"0", "1"} is just as likely a numeric indicator: stays gated. + result = infer_semantic_type("flag", _series(["0", "1", "0", "1"])) + assert result.semantic_type == "unknown" + + def test_infer_roles_gains_additive_columns(): df = pd.DataFrame( {