diff --git a/docs/peel.md b/docs/peel.md new file mode 100644 index 0000000..e8c27e1 --- /dev/null +++ b/docs/peel.md @@ -0,0 +1,110 @@ +--- +title: Peel — the output system +description: >- + Peel is freshdata's progressive-disclosure output system: a glance-level + answer, an inspect layer of evidence, and a full audit layer with zero + information loss, shared across the core API, semantic models, the AI + copilot, and parsing. +keywords: freshdata output, progressive disclosure, clean report, terminal rendering, notebook html, machine readable +--- + +# Peel — the output system + +Peel is how freshdata report objects present themselves. A result is fresh +produce: the **skin** tells you at a glance whether it is good, you can **peel** +to the flesh to inspect it, and the **core** is always intact. + +Every report — `CleanReport`, `ParseResult`, the experimental `CopilotReport` — +renders through the same three layers: + +| Layer | Name | What it answers | +|---|---|---| +| 1 | **Skin** (glance) | Did it succeed? What changed? Does anything need my attention? What's the one next step? | +| 2 | **Flesh** (inspect) | The evidence — per-column changes, ranked findings, semantic proposals, frame inventory. | +| 3 | **Core** (audit) | Every field, machine-readable, nothing dropped. | + +Peel never removes information; it only decides what you see first. Layer 3 is +the existing structured objects and their `to_dict()`/`to_json()` — display can +be turned off entirely and no data is lost. + +## Status and severity language + +Every result carries a **text status label** (colour and icons only reinforce +it, so output stays readable when piped, in CI, or for screen readers): + +`CLEAN` · `CHANGED` · `REVIEW` · `BLOCKED` · `PARTIAL` · `SKIPPED` · `FAILED` + +Findings in the attention list are ranked by one shared order: + +1. privacy & safety → 2. data-corruption risk → 3. policy/contract → 4. analysis +reliability → 5. cosmetic consistency + +so a high trust score can never bury a privacy or policy finding. + +## Seeing Peel output + +Peel is **opt-in** while it stabilizes; existing output is unchanged by default. + +```python +import freshdata as fd + +cleaned, report = fd.clean(df, return_report=True) + +report.show() # legacy behavior unchanged (inline HTML / temp file) +report.show(mode="standard") # Peel text: glance + attention + next step +report.show(mode="compact") # two lines, for pipelines +report.show(mode="verbose") # + per-column, semantic, and action detail +report.show(renderer="terminal") # styled panel when `rich` is installed +``` + +### Display modes + +| Mode | Use | +|---|---| +| `auto` | environment-aware: `standard` in a terminal, `compact` when piped | +| `compact` | one-screen, two lines | +| `standard` | glance + ranked attention + next step | +| `verbose` | full human-readable diagnostics | +| `debug` | + internal audit metadata | +| `json` | stable `to_dict()` on stdout | +| `plain` | no ANSI, ASCII icons | +| `silent` | render nothing | + +### Notebook + +`fd.set_display("peel")` (or `FRESHDATA_DISPLAY=peel`) switches the notebook +`_repr_html_` to the Peel card; `FRESHDATA_LEGACY_DISPLAY=1` forces the legacy +layout. Objects with only a Peel view (like `ParseResult`) always render as Peel. +Disclosure uses native `
`, so it works in static notebook exports with +no JavaScript. + +### Command line + +`freshdata clean` gains additive display flags; default output is unchanged: + +```bash +freshdata clean input.csv # unchanged (legacy summary) +freshdata clean input.csv --verbose # Peel verbose +freshdata clean input.csv -vv # Peel debug +freshdata clean input.csv --output-format json # report JSON to stdout +freshdata clean input.csv --no-color # no ANSI +freshdata clean input.csv --display peel # Peel standard +``` + +Display flags never change cleaning behavior. `fd.set_display(...)` sets +process-wide preferences; `NO_COLOR` and `FRESHDATA_NO_PREVIEWS` are honored. + +## Machine-readable access is unchanged + +`report.to_dict()` / `report.to_json()` schemas are frozen — additive only. +`report.actions`, `report.warnings`, `report.domain_findings`, `report.attention` +(the ranked queue) and every other attribute keep working. Peel is a rendering +layer over these objects, never a replacement for them. + +## Privacy + +Values shown on evidence cards and previews are escaped before HTML rendering +and never placed in HTML attributes. `fd.set_display(previews=False)` (or +`FRESHDATA_NO_PREVIEWS=1`) switches to schema-only display. `undo_log` is never +serialized, and provider credentials are never captured — only failure states +appear in the audit layer. diff --git a/mkdocs.yml b/mkdocs.yml index e3bbf78..f2b8bea 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -106,6 +106,7 @@ nav: - Cleaning engine: cleaning-engine.md - Validation (suites & contracts): validation.md - Interactive output: interactive.md + - Peel output system: peel.md - Decision-preserving workflow: decision-workflow.md - Scalable backends: backends.md - Backend fallback matrix: fallback-matrix.md diff --git a/pyproject.toml b/pyproject.toml index 67738ab..937c350 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,6 +91,10 @@ outofcore = [ cli = [ "pyyaml>=5.1", ] +# Styled terminal output (Peel). Everything renders as plain text without it. +rich = [ + "rich>=13", +] # Interactive output layer. The renderers produce self-contained HTML with zero # of these installed; these libraries *upgrade* tables/charts when present and # are never imported by `import freshdata`. diff --git a/src/freshdata/__init__.py b/src/freshdata/__init__.py index 2c72eb9..c0ffbde 100644 --- a/src/freshdata/__init__.py +++ b/src/freshdata/__init__.py @@ -77,6 +77,7 @@ ) from .profile import ColumnProfile, Profile from .quality import QualityDebtGate, evaluate_quality_debt +from .render.options import get_display, reset_display, set_display from .repairplan import ( FrameSignature, PlanDriftError, @@ -163,6 +164,9 @@ "apply_field_policy", "clean_text", "clean_text_value", + "get_display", + "reset_display", + "set_display", "validate_fields", "cdc_profile", "apply", diff --git a/src/freshdata/enterprise/cli.py b/src/freshdata/enterprise/cli.py index 2f42e41..e1b28b1 100644 --- a/src/freshdata/enterprise/cli.py +++ b/src/freshdata/enterprise/cli.py @@ -30,6 +30,62 @@ from .metrics import compute_trust_score +def _add_display_flags(parser: argparse.ArgumentParser) -> None: + """Additive Peel display flags (spec §11.3). Default output is unchanged.""" + group = parser.add_argument_group("display") + group.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="Peel diagnostics: -v verbose, -vv debug", + ) + group.add_argument( + "--output-format", + choices=("text", "json"), + default="text", + help="'json' prints the report as JSON to stdout", + ) + group.add_argument("--no-color", action="store_true", help="disable ANSI styling") + group.add_argument( + "--display", + choices=("legacy", "peel"), + default="legacy", + help="'peel' opts into the Peel layout for this run", + ) + + +def _emit_report(report: Any, args: argparse.Namespace, legacy_text: str) -> None: + """Print a clean report honoring the display flags. + + With no display flags the legacy text is printed verbatim (backward + compatible); ``--output-format json``/``--verbose``/``--display peel`` + opt into Peel output. + """ + fmt = getattr(args, "output_format", "text") + verbose = getattr(args, "verbose", 0) + display = getattr(args, "display", "legacy") + + if fmt == "json": + print(json.dumps(report.to_dict(), default=str, indent=2)) + return + if verbose == 0 and display == "legacy": + print(legacy_text) + return + + from ..render.normalize import normalize + from ..render.options import get_display + from ..render.terminal import render_terminal_text + + mode = "debug" if verbose >= 2 else "verbose" if verbose == 1 else "standard" + color = "never" if getattr(args, "no_color", False) else "auto" + try: + options = get_display(mode=mode, color=color) + print(render_terminal_text(normalize(report), options)) + except Exception: + print(legacy_text) # display must never break the command + + def _infer_format(path: str) -> str: low = path.lower() if low.endswith((".parquet", ".pq")): @@ -183,7 +239,7 @@ def cmd_clean(args: argparse.Namespace) -> int: if args.lineage: result.lineage.emit(args.lineage) if not args.quiet: - print(result.summary()) + _emit_report(result.clean_report, args, result.summary()) for event in result.clean_report.fallback_events: if event.get("fallback_step") == "semantic": print( @@ -237,7 +293,7 @@ def _cmd_clean_engine(args: argparse.Namespace) -> int: with open(args.report, "w", encoding="utf-8") as fh: json.dump(report.to_dict(), fh, indent=2, default=str) if not args.quiet: - print(report.summary()) + _emit_report(report, args, report.summary()) if report.backend_differences: print( f"\n{len(report.backend_differences)} backend difference(s) recorded " @@ -627,6 +683,7 @@ def build_parser() -> argparse.ArgumentParser: ) clean.add_argument("--actor", help="who ran this (recorded in lineage)") clean.add_argument("--quiet", action="store_true") + _add_display_flags(clean) clean.add_argument( "--context-file", metavar="PATH", diff --git a/src/freshdata/experimental/ai_copilot.py b/src/freshdata/experimental/ai_copilot.py index c20d874..ccedb15 100644 --- a/src/freshdata/experimental/ai_copilot.py +++ b/src/freshdata/experimental/ai_copilot.py @@ -66,6 +66,7 @@ from ..enterprise.config import ClusterConfig, MaskingRule from ..enterprise.metrics import TrustScore, compute_trust_score from ..enterprise.privacy import PIIDetectionConfig, anonymize, detect_pii +from ..render.mixins import HtmlReprMixin __all__ = [ "CleaningPlan", @@ -172,7 +173,7 @@ def __str__(self) -> str: @dataclass(frozen=True) -class CopilotReport: +class CopilotReport(HtmlReprMixin): """Everything one :func:`analyze_dataset` run produced. ``summary``, ``cleaning_plan``, and ``recommended_code`` are all directly @@ -194,6 +195,18 @@ class CopilotReport: #: default deterministic path or when the provider call failed). narrative: str | None = None + _render_kind = "copilot" + + @property + def attention(self) -> tuple: + """The ranked attention queue (privacy/policy first), as ``AttentionItem``\\s. + + Terminal, notebook, and programmatic callers all read the same order. + """ + from ..render import normalize # noqa: PLC0415 + + return normalize.normalize(self).attention + def to_dict(self) -> dict[str, Any]: return { "goal": self.goal, diff --git a/src/freshdata/parsers/base.py b/src/freshdata/parsers/base.py index 54766f4..92a1626 100644 --- a/src/freshdata/parsers/base.py +++ b/src/freshdata/parsers/base.py @@ -17,11 +17,13 @@ import pandas as pd +from ..render.mixins import HtmlReprMixin + _MAX_XML_BYTES = 10 * 1024 * 1024 @dataclass -class ParseResult: +class ParseResult(HtmlReprMixin): """The structural output of a :class:`Parser`. Attributes @@ -46,6 +48,21 @@ class ParseResult: metadata: dict[str, Any] = field(default_factory=dict) warnings: list[str] = field(default_factory=list) + _render_kind = "parse" + + def summary(self) -> str: + """Plain-text Peel summary — parsing is structural only, so this makes + the parsed/validated/cleaned distinction explicit (spec §9).""" + # Deferred: render.normalize imports back into freshdata internals; a + # module-level import here would be circular at package-init time. + from ..render import normalize, plain # noqa: PLC0415 + from ..render.options import get_display # noqa: PLC0415 + + return plain.render_plain(normalize.normalize(self), get_display(mode="standard")) + + def __str__(self) -> str: + return self.summary() + @property def frame(self) -> pd.DataFrame: """The single frame, for formats that yield exactly one.""" diff --git a/src/freshdata/render/_semantic.py b/src/freshdata/render/_semantic.py new file mode 100644 index 0000000..573c598 --- /dev/null +++ b/src/freshdata/render/_semantic.py @@ -0,0 +1,151 @@ +"""Semantic-evidence helpers for the Peel normalizer (spec §7). + +Semantic decisions ride into a :class:`CleanReport` as ``Action``\\s with +``step == "semantic"`` and a ``metadata`` dict carrying the proposal payload +(``raw_value``, ``proposed_value``, signed ``evidence`` entries, ``backend``, +optional ``calibrated_confidence``/``model_evidence``/``memory_key``). This +module turns those into display-neutral rows and a plain-language confidence +phrase, keeping every raw field for the audit layer. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from .view import confidence_phrase + +if TYPE_CHECKING: # pragma: no cover - typing only + from ..report import Action, CleanReport + + +def is_semantic(action: Action) -> bool: + return action.step == "semantic" or "evidence" in (action.metadata or {}) + + +def _decision(action: Action) -> str: + """One of ``applied`` / ``review`` / ``ambiguous`` from the Action status.""" + if action.status == "suggested": + return "review" + if action.status == "skipped": + return "ambiguous" + return "applied" + + +def _source(action: Action) -> str: + """Plain-language provenance: rules / cleaning memory / model.""" + meta = action.metadata or {} + if meta.get("model_evidence"): + return "model" + if action.memory_influenced or meta.get("memory_key"): + return "cleaning memory" + return "rules" + + +def _confidence(action: Action) -> tuple[float, bool]: + """Effective confidence and whether it was independently calibrated.""" + meta = action.metadata or {} + calibrated = "calibrated_confidence" in meta + value = meta.get("calibrated_confidence", action.confidence) + return float(value or 0.0), calibrated + + +def confidence_label(action: Action) -> str: + """``"moderate evidence (0.84)"`` — number and plain phrase together.""" + value, calibrated = _confidence(action) + phrase = confidence_phrase(value, ambiguous=_decision(action) == "ambiguous") + prefix = "" if calibrated else "~" + return f"{phrase} ({prefix}{value:.2f})" + + +def attention_text(action: Action) -> str: + """Glance-layer sentence for a semantic review/ambiguous item.""" + meta = action.metadata or {} + raw = meta.get("raw_value") + proposed = meta.get("proposed_value") + if _decision(action) == "ambiguous": + change = f"'{raw}' → no change made" if raw is not None else action.description + elif raw is not None and proposed is not None: + change = f"'{raw}' → '{proposed}'" + else: + change = action.description + return f"{change} — {confidence_label(action)}" + + +def _evidence_summary(action: Action) -> str: + """Signed evidence as one readable line, strongest first. + + Raw ``{kind, detail, weight}`` entries stay intact in the action metadata + (audit layer); this is the inspect-layer digest. + """ + signals = list((action.metadata or {}).get("evidence", ())) + signals.sort(key=lambda s: -abs(float(s.get("weight", 0.0)))) + parts = [] + for signal in signals: + weight = float(signal.get("weight", 0.0)) + detail = str(signal.get("detail", signal.get("kind", ""))) + parts.append(f"{weight:+.2f} {detail}") + return "; ".join(parts) + + +def _max_evidence_weight(action: Action) -> float: + signals = (action.metadata or {}).get("evidence", ()) + weights = [abs(float(s.get("weight", 0.0))) for s in signals] + return max(weights) if weights else 0.0 + + +def semantic_rows(rep: CleanReport) -> list[dict[str, Any]]: + """Semantic proposals as display rows, review/ambiguous before applied.""" + order = {"ambiguous": 0, "review": 1, "applied": 2} + rows = [] + for action in rep.actions: + if not is_semantic(action): + continue + meta = action.metadata or {} + decision = _decision(action) + rows.append( + { + "decision": decision, + "column": action.column or "", + "change": attention_text(action).split(" — ")[0], + "confidence": confidence_label(action), + "source": _source(action), + "reversible": action.reversible, + "reason": action.rationale, + "backend": meta.get("backend", ""), + "evidence": _evidence_summary(action), + "_order": (order[decision], -_max_evidence_weight(action), action.column or ""), + } + ) + rows.sort(key=lambda r: r["_order"]) + for row in rows: + del row["_order"] + return rows + + +def coverage_note(rep: CleanReport) -> str: + """Plain-language line naming which evidence sources actually ran.""" + sources = {_source(a) for a in rep.actions if is_semantic(a)} + if not sources: + return "" + ran = ", ".join(s for s in ("rules", "cleaning memory", "model") if s in sources) + skipped = [ + e.get("fallback_step") or e.get("backend") + for e in rep.fallback_events + if "semantic" in str(e.get("fallback_step", "")).lower() + or "model" in str(e.get("backend", "")).lower() + ] + note = f"checked with {ran}" + if skipped: + note += ( + "; FreshData continued without the optional model " + f"({', '.join(str(s) for s in skipped)})" + ) + return note + + +def count_by_decision(rep: CleanReport) -> dict[str, int]: + counts = {"applied": 0, "review": 0, "ambiguous": 0} + for action in rep.actions: + if is_semantic(action): + counts[_decision(action)] += 1 + return counts diff --git a/src/freshdata/render/_vocabulary.py b/src/freshdata/render/_vocabulary.py new file mode 100644 index 0000000..7b0f09f --- /dev/null +++ b/src/freshdata/render/_vocabulary.py @@ -0,0 +1,48 @@ +"""Peel's plain-language vocabulary (spec §14). + +Display layers 1-2 speak user language; the audit layer keeps exact technical +terms. This module is the single place display wording is defined so every +renderer stays consistent. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: # pragma: no cover - typing only + from ..report import Action + +#: step name → plain-language verb phrase, used when summarizing per column. +_STEP_PHRASES = { + "missing": "filled missing values", + "impute": "filled missing values", + "outliers": "extreme values adjusted", + "duplicates": "removed duplicate rows", + "fix_dtypes": "fixed value types", + "whitespace": "trimmed whitespace", + "semantic": "standardized values", +} + +#: audit/internal term → display phrase (spec §14.2), applied to prose. +TERMS = { + "backend abstained": "no safe match was found", + "fallback event": "FreshData continued without the optional engine/model", + "calibration unavailable": "confidence could not be independently adjusted", + "constraint violation": "this value breaks a rule you defined", + "residual values": "values not resolved by earlier checks", + "not materialized": "result kept in the engine", +} + + +def plain_step(action: Action) -> str: + """A short plain-language phrase for *action*, for per-column summaries. + + Falls back to the action's own description — descriptions are already + human sentences; the map only replaces the jargon-heavy step families. + """ + phrase = _STEP_PHRASES.get(action.step) + if phrase is None: + return action.description + if action.count: + return f"{phrase} ({action.count:,})" + return phrase diff --git a/src/freshdata/render/mixins.py b/src/freshdata/render/mixins.py index d35a44e..35bac07 100644 --- a/src/freshdata/render/mixins.py +++ b/src/freshdata/render/mixins.py @@ -18,10 +18,34 @@ class HtmlReprMixin: _render_kind: str = "" def to_html(self) -> str: - """Return a self-contained HTML fragment for this report object.""" + """Return a self-contained HTML fragment for this report object. + + Kinds with a legacy renderer keep it by default; ``fd.set_display("peel")`` + (or ``FRESHDATA_DISPLAY=peel``) switches those to the Peel card, and + ``FRESHDATA_LEGACY_DISPLAY=1`` forces legacy. Kinds that only have a + Peel normalizer (no legacy layout to preserve) always render as Peel. + """ + import os + from . import renderers - return renderers.render(self, self._render_kind) + kind = self._render_kind + has_legacy = kind in renderers._DISPATCH + legacy_forced = bool(os.environ.get("FRESHDATA_LEGACY_DISPLAY")) + + if not (has_legacy and legacy_forced): + from .options import get_display + + want_peel = get_display().style == "peel" or not has_legacy + if want_peel: + try: + from . import normalize, notebook + + return notebook.render_notebook(normalize.normalize(self)) + except KeyError: + if not has_legacy: + raise # nothing else can render this kind + return renderers.render(self, kind) def _repr_html_(self) -> str | None: """Rich display hook for Jupyter; falls back to text on any failure.""" @@ -30,13 +54,24 @@ def _repr_html_(self) -> str | None: except Exception: # pragma: no cover - display must never raise return None - def show(self) -> Any: - """Display in a notebook, or write an HTML file and return its path. + def show(self, mode: str | None = None, *, renderer: str | None = None) -> Any: + """Display this report. + + With no arguments the behavior is unchanged: in Jupyter/IPython the + HTML renders inline; outside a notebook a standalone ``.html`` file is + written to a temp location and its path returned. - In Jupyter/IPython this renders inline. Outside a notebook it writes a - standalone ``.html`` file to a temp location and returns the path, so the - same call works from scripts and the REPL. + ``mode`` (``"compact"``/``"standard"``/``"verbose"``/``"debug"``/ + ``"json"``/``"plain"``/``"silent"``) or ``renderer="terminal"`` selects + the Peel text output instead; ``renderer="notebook"`` keeps the HTML + path. Display never raises: on any failure the Peel path falls back to + the object's ``summary()``/``repr``. """ + if mode is not None or renderer == "terminal": + text = self._peel_text(mode, styled=renderer == "terminal") + if text: + print(text) + return None html = self.to_html() try: from ._optional import require @@ -61,6 +96,26 @@ def show(self) -> Any: print(f"freshdata: wrote {kind} report to {path}") return path + def _peel_text(self, mode: str | None, *, styled: bool = False) -> str: + """Peel text rendering with the never-raise fallback chain.""" + try: + from . import normalize, plain, terminal + from .options import get_display + + options = get_display(mode=mode) if mode is not None else get_display() + view = normalize.normalize(self) + if styled: + return terminal.render_terminal_text(view, options).rstrip("\n") + return plain.render_plain(view, options) + except Exception: + summary = getattr(self, "summary", None) + if callable(summary): + try: + return str(summary()) + except Exception: # pragma: no cover - summary must not raise + pass + return repr(self) + class SimpleHtmlReport(HtmlReprMixin): """Base for report objects that build their own HTML from the primitives. diff --git a/src/freshdata/render/normalize.py b/src/freshdata/render/normalize.py new file mode 100644 index 0000000..a117ca7 --- /dev/null +++ b/src/freshdata/render/normalize.py @@ -0,0 +1,617 @@ +"""Report objects → :class:`~freshdata.render.view.PeelView`. + +One pure function per ``_render_kind``. Normalizers read only the report +object — never a DataFrame — and are deterministic: the same report always +yields the same view, including attention ordering and finding ids. + +Finding-id prefixes are per source so terminal, notebook, and programmatic +access all name the same thing: + +``W`` report warnings · ``R`` recommendations · ``S`` suggested/review actions +· ``D`` domain findings · ``C`` contract findings · ``F`` fallback events +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Callable + +from . import _semantic +from ._vocabulary import plain_step +from .view import AttentionItem, Metric, PeelView, Section, rank_attention + +if TYPE_CHECKING: # pragma: no cover - typing only + from ..report import CleanReport + +#: Normalizers by ``_render_kind``; extended by later layers (parse, copilot). +_NORMALIZERS: dict[str, Callable[[Any], PeelView]] = {} + + +def register_normalizer(kind: str, fn: Callable[[Any], PeelView]) -> None: + """Register *fn* as the normalizer for ``_render_kind == kind``. + + Third-party report objects use this the same way built-ins do (spec §12.3). + """ + _NORMALIZERS[kind] = fn + + +def normalize(obj: Any) -> PeelView: + """Return the PeelView for *obj*, dispatching on its ``_render_kind``.""" + kind = getattr(obj, "_render_kind", "") + fn = _NORMALIZERS.get(kind) + if fn is None: + raise KeyError(f"no Peel normalizer registered for kind {kind!r}") + return fn(obj) + + +# -- clean_report ------------------------------------------------------------ + + +def _fmt_duration(seconds: float) -> str: + return f"{seconds:.1f}s" if seconds >= 0.05 else f"{seconds * 1000:.0f}ms" + + +def _headline(rep: CleanReport) -> str: + if not rep.materialized: + return ( + f"{rep.rows_before:,} rows in · result kept in the engine " + f"(counts not computed to avoid a full scan) · {_fmt_duration(rep.duration_seconds)}" + ) + parts = [f"{rep.rows_after:,} of {rep.rows_before:,} rows kept"] + if rep.cols_after != rep.cols_before: + parts.append(f"{rep.cols_before:,} → {rep.cols_after:,} columns") + else: + parts.append(f"{rep.cols_after:,} columns") + parts.append(f"{rep.cells_changed:,} cells changed") + parts.append(_fmt_duration(rep.duration_seconds)) + return " · ".join(parts) + + +def _statuses(rep: CleanReport, attention: tuple[AttentionItem, ...]) -> tuple[str, ...]: + statuses = ["CHANGED" if bool(rep) else "CLEAN"] + if any(a.severity in ("error", "warning", "review") for a in attention): + statuses.append("REVIEW") + if not rep.materialized: + statuses.append("PARTIAL") + return tuple(statuses) + + +def _metrics(rep: CleanReport) -> tuple[Metric, ...]: + if not rep.materialized: + return (Metric("rows in", f"{rep.rows_before:,}"),) + metrics = [ + Metric( + "missing", + f"{rep.missing_after:,}", + before=f"{rep.missing_before:,}", + after=f"{rep.missing_after:,}", + ) + ] + if rep.duplicates_removed: + metrics.append(Metric("duplicates", f"-{rep.duplicates_removed:,}")) + if rep.outliers_handled: + metrics.append(Metric("extreme values adjusted", f"{rep.outliers_handled:,}")) + if rep.columns_preserved: + metrics.append(Metric("protected", f"{len(rep.columns_preserved)}")) + if rep.columns_dropped: + metrics.append(Metric("columns dropped", f"{len(rep.columns_dropped)}")) + return tuple(metrics) + + +def _clean_attention(rep: CleanReport) -> tuple[AttentionItem, ...]: + items: list[AttentionItem] = [] + + for i, message in enumerate(rep.warnings, 1): + items.append( + AttentionItem( + id=f"W{i}", + severity="warning", + subject="", + text=message, + domain="reliability", + detail={"source": "warnings"}, + ) + ) + for i, message in enumerate(rep.recommendations, 1): + items.append( + AttentionItem( + id=f"R{i}", + severity="review", + subject="", + text=message, + domain="reliability", + detail={"source": "recommendations"}, + ) + ) + + suggested = [ + a + for a in rep.actions + if a.status == "suggested" + or (a.human_review and a.count) + # ambiguous semantic proposals made no change but still want a human's eye + or (_semantic.is_semantic(a) and a.status == "skipped") + ] + for i, action in enumerate(suggested, 1): + if _semantic.is_semantic(action): + text = _semantic.attention_text(action) + else: + text = f"{action.description} — held for your review" + items.append( + AttentionItem( + id=f"S{i}", + severity="review", + subject=action.column or "", + text=text, + domain="corruption" if action.risk == "high" else "reliability", + count=action.count, + detail={"source": "actions", "action": rep._action_dict(action)}, + ) + ) + + for i, finding in enumerate(rep.domain_findings, 1): + if finding.get("status") != "violated": + continue + severity = "error" if finding.get("severity") == "error" else "warning" + items.append( + AttentionItem( + id=f"D{i}", + severity=severity, + subject=str(finding.get("column") or finding.get("rule") or ""), + text=str(finding.get("message") or finding.get("rule") or "domain rule violated"), + domain="policy", + count=int(finding.get("count") or 0), + detail={"source": "domain_findings", "finding": finding}, + ) + ) + + cv = rep.contract_violations + if cv is not None and not cv.get("passed", True): + for i, finding in enumerate(cv.get("findings", []), 1): + if finding.get("status") == "passed": + continue + severity = "error" if finding.get("status") == "failed" else "warning" + items.append( + AttentionItem( + id=f"C{i}", + severity=severity, + subject=str(finding.get("column") or ""), + text=( + f"contract '{cv.get('baseline_name')}': " + f"{finding.get('message') or finding.get('check_id')}" + ), + domain="policy", + detail={"source": "contract_violations", "finding": finding}, + ) + ) + + for i, event in enumerate(rep.fallback_events, 1): + items.append( + AttentionItem( + id=f"F{i}", + severity="info", + subject=str(event.get("fallback_step") or ""), + text=( + "FreshData continued without the optional engine step " + f"({event.get('backend')}: {event.get('fallback_reason')})" + ), + domain="reliability", + detail={"source": "fallback_events", "event": event}, + ) + ) + + return rank_attention(items) + + +def _next_step(rep: CleanReport, attention: tuple[AttentionItem, ...]) -> str | None: + if not attention or attention[0].severity == "info": + return None + top = attention[0] + source = top.detail.get("source") + if source == "contract_violations": + return "report.contract_violations # review contract failures" + if source == "domain_findings": + return "report.domain_findings # review domain rule violations" + if source == "actions": + return "fd.suggest_plan(df) # review and approve the suggested changes" + return "fd.explain_clean(df) # see the evidence behind each decision" + + +def _column_rows(rep: CleanReport) -> list[dict[str, Any]]: + """Per-column aggregation of the action log, most-changed first.""" + by_column: dict[str, dict[str, Any]] = {} + risk_rank = {"low": 0, "medium": 1, "high": 2} + for action in rep.actions: + key = action.column if action.column is not None else "(table)" + row = by_column.setdefault( + key, {"column": key, "changed": 0, "what": [], "risk": "low"} + ) + row["changed"] += action.count + row["what"].append(plain_step(action)) + if risk_rank.get(action.risk, 0) > risk_rank.get(row["risk"], 0): + row["risk"] = action.risk + for name in rep.columns_preserved: + row = by_column.setdefault( + name, {"column": name, "changed": 0, "what": [], "risk": "low"} + ) + row["protected"] = True + rows = list(by_column.values()) + rows.sort(key=lambda r: (-r["changed"], r["column"])) + for row in rows: + row["what"] = "; ".join(dict.fromkeys(row["what"])) or ( + "protected — left untouched" if row.get("protected") else "" + ) + return rows + + +def _audit_rows(rep: CleanReport) -> list[dict[str, Any]]: + keys = ( + "backend", + "materialized", + "duration_seconds", + "memory_before", + "memory_after", + "fallback_events", + "backend_differences", + "stage_timings", + "domain", + "domain_trust_score", + "domain_repairs", + "streaming", + "profile_replay", + "source_provenance", + "contract_violations", + "decisions_hash", + ) + rows = [] + for key in keys: + value = getattr(rep, key) + if value not in (None, [], {}): + rows.append({"field": key, "value": value}) + return rows + + +def normalize_clean_report(rep: CleanReport) -> PeelView: + """Normalize a :class:`~freshdata.report.CleanReport` (spec §6).""" + attention = _clean_attention(rep) + banner = None + if not rep.materialized: + banner = ( + "PARTIAL — result kept in the engine; row counts not computed " + "to avoid a full scan. Call .fetchdf()/.collect() to pull rows." + ) + sections_list = [ + Section( + "columns", + "Column changes", + lambda: _column_rows(rep), + count=len({a.column for a in rep.actions if a.column is not None}), + ), + ] + semantic_counts = _semantic.count_by_decision(rep) + n_semantic = sum(semantic_counts.values()) + if n_semantic: + note = _semantic.coverage_note(rep) + title = ( + f"Semantic proposals ({semantic_counts['applied']} applied · " + f"{semantic_counts['review']} review · {semantic_counts['ambiguous']} no change)" + ) + if note: + title = f"{title} — {note}" + sections_list.append( + Section("semantic", title, lambda: _semantic.semantic_rows(rep), count=n_semantic) + ) + sections_list.extend( + [ + Section( + "actions", + "All actions", + lambda: [rep._action_dict(a) for a in rep.actions], + count=len(rep.actions), + ), + Section("audit", "Audit", lambda: _audit_rows(rep), count=len(_audit_rows(rep))), + ] + ) + sections = tuple(sections_list) + return PeelView( + kind="clean_report", + status=_statuses(rep, attention), + headline=_headline(rep), + metrics=_metrics(rep), + attention=attention, + next_step=_next_step(rep, attention), + sections=sections, + audit_ref=rep, + banner=banner, + ) + + +register_normalizer("clean_report", normalize_clean_report) + + +# -- parse ------------------------------------------------------------------- + + +def _frame_status(name: str, n_rows: int, warnings: list[str]) -> tuple[str, str]: + """(machine, human) status for one parsed frame.""" + hits = [w for w in warnings if _warning_mentions(w, name)] + if n_rows == 0: + return "unsupported", "unsupported items found" + if hits: + return "warnings", f"{len(hits)} warning(s)" + return "ready", "ready" + + +def _warning_mentions(warning: str, frame: str) -> bool: + low = warning.lower() + return low.startswith(f"{frame.lower()}:") or f"{frame.lower()} " in low + + +def _stage_ladder() -> str: + # parse() only reaches the first rung; later stages run under fd.clean(...). + return "parsed ✓ · validated — · cleaned — · reference —" + + +def normalize_parse_result(res: Any) -> PeelView: + """Normalize a :class:`~freshdata.parsers.base.ParseResult` (spec §9). + + Reads only frame *row counts* (``len``) — never frame contents. + """ + warnings = list(res.warnings) + frame_rows = {name: len(df) for name, df in res.frames.items()} + total_rows = sum(frame_rows.values()) + empty_frames = [n for n, r in frame_rows.items() if r == 0] + partial = bool(warnings or empty_frames) + + status = ("PARTIAL",) if partial else ("CLEAN",) + headline = ( + f"{len(res.frames)} frame(s) · {total_rows:,} rows total" + + (f" · {len(warnings)} warning(s)" if warnings else "") + ) + + metrics = [Metric("stage", _stage_ladder())] + if res.suggested_domain: + metrics.append(Metric("suggested domain", f"{res.suggested_domain} (advisory)")) + + items: list[AttentionItem] = [] + wi = 0 + for name in res.frames: + n_rows = frame_rows[name] + machine, _human = _frame_status(name, n_rows, warnings) + if machine == "unsupported": + wi += 1 + items.append( + AttentionItem( + id=f"P{wi}", + severity="warning", + subject=name, + text="0 rows — unsupported items were skipped and preserved in warnings", + domain="reliability", + detail={"source": "frames", "frame": name}, + ) + ) + for warning in warnings: + if any(_warning_mentions(warning, n) and frame_rows[n] == 0 for n in res.frames): + continue # already represented by the frame's unsupported item + wi += 1 + items.append( + AttentionItem( + id=f"P{wi}", + severity="warning", + subject="", + text=warning, + domain="reliability", + detail={"source": "warnings"}, + ) + ) + attention = rank_attention(items) + + banner = None + if partial: + banner = ( + "PARTIAL — this data has been read, not checked. " + "Structural parsing succeeded; validation and cleaning have not run." + ) + + next_step = None + if res.suggested_domain and res.frames: + first = next(iter(res.frames)) + next_step = f'fd.clean(result.frames["{first}"], domain="{res.suggested_domain}")' + + def frame_rows_section() -> list[dict[str, Any]]: + rows = [] + for name in res.frames: + machine, human = _frame_status(name, frame_rows[name], warnings) + rows.append({"frame": name, "rows": frame_rows[name], "status": human}) + return rows + + sections = ( + Section("frames", "Frames", frame_rows_section, count=len(res.frames)), + Section( + "warnings", + "All warnings", + lambda: [{"warning": w} for w in warnings], + count=len(warnings), + ), + Section( + "metadata", + "Format metadata", + lambda: [{"key": k, "value": v} for k, v in dict(res.metadata).items()], + count=len(res.metadata), + ), + ) + + return PeelView( + kind="parse", + status=status, + headline=headline, + metrics=tuple(metrics), + attention=attention, + next_step=next_step, + sections=sections, + audit_ref=res, + banner=banner, + ) + + +register_normalizer("parse", normalize_parse_result) + + +# -- copilot ----------------------------------------------------------------- + +_PROBLEM_SEVERITY = {"high": "error", "medium": "warning", "low": "info"} +_PROBLEM_DOMAIN = {"high": "corruption", "medium": "reliability", "low": "cosmetic"} + + +def normalize_copilot_report(rep: Any) -> PeelView: + """Normalize a :class:`~freshdata.experimental.ai_copilot.CopilotReport`. + + The attention queue is ranked by the shared comparator, so privacy and + policy findings always sit above data-quality ones — a high trust score can + never bury them (spec §8). IDs (``A1``…) are assigned after ranking. + """ + items: list[AttentionItem] = [] + + if rep.pii_warning: + columns = rep.audit.get("pii_columns") or rep.audit.get("pii_entities_found") + items.append( + AttentionItem( + id="_pii", + severity="error", + subject="", + text=str(rep.pii_warning), + domain="privacy", + detail={"source": "pii_warning", "pii_columns": columns}, + ) + ) + for i, violation in enumerate(rep.policy_violations): + text = ( + violation.to_dict() + if hasattr(violation, "to_dict") + else {"detail": str(violation)} + ) + message = text.get("message") or text.get("detail") or str(violation) + items.append( + AttentionItem( + id=f"_pol{i}", + severity="error", + subject=str(text.get("column") or ""), + text=f"policy: {message}", + domain="policy", + detail={"source": "policy_violations", "violation": text}, + ) + ) + for i, problem in enumerate(rep.problems): + items.append( + AttentionItem( + id=f"_prob{i}", + severity=_PROBLEM_SEVERITY.get(problem.severity, "warning"), + subject=str(problem.column or ""), + text=str(problem.detail), + domain=_PROBLEM_DOMAIN.get(problem.severity, "reliability"), + count=int(problem.count or 0), + detail={"source": "problems", "problem": problem.to_dict()}, + ) + ) + + ranked = rank_attention(items) + attention = tuple( + AttentionItem( + id=f"A{i}", + severity=item.severity, + subject=item.subject, + text=item.text, + domain=item.domain, + count=item.count, + detail=item.detail, + ) + for i, item in enumerate(ranked, 1) + ) + + provider_error = rep.audit.get("provider_error") + status: list[str] = ["REVIEW"] if attention else ["CLEAN"] + if provider_error: + status.append("PARTIAL") + + trust = rep.trust + headline = ( + f'goal "{rep.goal}" · trust {trust.overall:.0f}/100 ({trust.grade}) · ' + f"{len(attention)} finding(s)" + ) + + pii_state = ( + "PII found — masked before the model saw any data" + if rep.pii_warning + else "no PII detected — nothing was masked" + ) + metrics = ( + Metric("privacy", pii_state), + Metric("trust", f"{trust.overall:.0f}/100 ({trust.grade}) — data quality only"), + Metric( + "dimensions", + f"completeness {trust.completeness:.0f} · validity {trust.validity:.0f} · " + f"uniqueness {trust.uniqueness:.0f} · consistency {trust.consistency:.0f}", + ), + ) + + banner = "Experimental API — review all generated code before running." + if provider_error: + banner += ( + " Model narrative unavailable (provider error) — the deterministic " + "findings above are complete." + ) + + next_step = None + if rep.cleaning_plan.steps: + next_step = str(rep.cleaning_plan.steps[0].tool) + + def plan_rows() -> list[dict[str, Any]]: + return [ + {"order": s.order, "action": s.action, "why": s.rationale, "tool": s.tool} + for s in rep.cleaning_plan.steps + ] + + def trust_columns() -> list[dict[str, Any]]: + rows = [] + for col in getattr(trust, "columns", ()): # ColumnTrust tuple + if hasattr(col, "to_dict"): + rows.append(dict(col.to_dict())) + return rows + + sections = ( + Section("plan", "Cleaning plan", plan_rows, count=len(rep.cleaning_plan.steps)), + Section( + "code", + "Recommended code (machine-generated — review before running)", + lambda: [{"code": rep.recommended_code}], + count=1 if rep.recommended_code else 0, + ), + Section("trust_columns", "Per-column trust", trust_columns, + count=len(getattr(trust, "columns", ()))), + Section( + "model_context", + "Model context (masked — the only payload a provider sees)", + lambda: [{"key": k, "value": v} for k, v in dict(rep.model_context).items()], + count=len(rep.model_context), + ), + Section( + "audit", + "Audit", + lambda: [{"field": k, "value": v} for k, v in dict(rep.audit).items()], + count=len(rep.audit), + ), + ) + + return PeelView( + kind="copilot", + status=tuple(status), + headline=headline, + metrics=metrics, + attention=attention, + next_step=next_step, + sections=sections, + audit_ref=rep, + banner=banner, + ) + + +register_normalizer("copilot", normalize_copilot_report) diff --git a/src/freshdata/render/notebook.py b/src/freshdata/render/notebook.py new file mode 100644 index 0000000..0003895 --- /dev/null +++ b/src/freshdata/render/notebook.py @@ -0,0 +1,153 @@ +"""Peel notebook renderer — PeelView → self-contained HTML (spec §10). + +Opt-in for now: ``fd.set_display("peel")`` (or ``FRESHDATA_DISPLAY=peel``) +switches ``_repr_html_``/``to_html`` to this renderer for objects that have a +Peel normalizer; everything else keeps the legacy layout. Disclosure uses +native ``
`` so static exports and no-JS viewers keep full access. + +Everything user-provided is escaped via :func:`freshdata.render.html.esc`; +values never land in HTML attributes. +""" + +from __future__ import annotations + +import json + +from . import html as H +from .options import RenderOptions, get_display +from .view import AttentionItem, PeelView, Section + +#: severity → chip color (labels always accompany color; see spec §14.1). +_SEVERITY_COLORS = { + "error": "#cf222e", + "warning": "#9a6700", + "review": "#0969da", + "info": "#57606a", +} + +_STATUS_COLORS = { + "CLEAN": "#1a7f37", + "CHANGED": "#0969da", + "REVIEW": "#9a6700", + "BLOCKED": "#cf222e", + "PARTIAL": "#9a6700", + "SKIPPED": "#57606a", + "FAILED": "#cf222e", +} + +#: Cap on rows rendered per inspect section; the full data stays reachable +#: through to_dict()/to_json() and verbose text modes (spec §16). +_MAX_SECTION_ROWS = 50 + + +def _status_chips(view: PeelView) -> str: + chips = "".join( + H.badge(status, _STATUS_COLORS.get(status, "#57606a")) for status in view.status + ) + return f'
{chips}
' + + +def _metric_strip(view: PeelView) -> str: + if not view.metrics: + return "" + items = [] + for metric in view.metrics: + if metric.before is not None: + items.append((metric.label, f"{metric.before} → {metric.after}")) + else: + items.append((metric.label, metric.value)) + return H.scorecards(items) + + +def _attention_html(items: tuple[AttentionItem, ...]) -> str: + if not items: + return '

nothing needs review

' + rows = [] + for item in items: + chip = H.badge(item.severity.capitalize(), _SEVERITY_COLORS.get(item.severity, "#57606a")) + subject = f"{H.esc(item.subject)} " if item.subject else "" + rows.append( + f'
  • {chip} {subject}{H.esc(item.text)} ' + f'[{H.esc(item.id)}]
  • ' + ) + return ( + f"

    Needs attention ({len(items)})

    " + f'" + ) + + +def _next_step_html(view: PeelView) -> str: + if not view.next_step: + return "" + return ( + '

    next → ' + f"{H.esc(view.next_step)}

    " + ) + + +def _banner_html(view: PeelView) -> str: + if not view.banner: + return "" + return ( + '

    ' + f"{H.esc(view.banner)}

    " + ) + + +def _section_html(section: Section) -> str: + rows = section.rows() + shown = rows[:_MAX_SECTION_ROWS] + columns: list[str] = [] + for row in shown: + for key in row: + if key not in columns: + columns.append(key) + body = H.table(columns, [[_cell(row.get(key)) for key in columns] for row in shown]) + if len(rows) > len(shown): + body += ( + f'

    showing {len(shown)} of {len(rows)} — ' + "use report.to_json() or " + 'report.show(mode="verbose") for all

    ' + ) + summary = f"{H.esc(section.title)} ({section.count})" + return H.collapsible(summary, body) + + +def _cell(value: object) -> str: + if value in (None, "", [], {}): + return "" + if isinstance(value, float): + return f"{value:g}" + if isinstance(value, (dict, list)): + return json.dumps(value, default=str) + return str(value) + + +def _export_html(view: PeelView) -> str: + report = view.audit_ref + if report is None or not hasattr(report, "to_dict"): + return "" + try: + return H.json_download(f"freshdata_{view.kind}.json", report.to_dict()) + except Exception: # pragma: no cover - export must never break display + return "" + + +def render_notebook(view: PeelView, options: RenderOptions | None = None) -> str: + """Render *view* as a self-contained Peel HTML fragment.""" + options = options or get_display() + parts = [ + _status_chips(view), + _banner_html(view), + _metric_strip(view), + _attention_html(view.attention), + _next_step_html(view), + ] + parts.extend(_section_html(section) for section in view.sections) + parts.append(_export_html(view)) + from .plain import _TITLES + + title = _TITLES.get(view.kind, f"freshdata {view.kind}") + return H.document(title, *parts, subtitle=view.headline) diff --git a/src/freshdata/render/options.py b/src/freshdata/render/options.py new file mode 100644 index 0000000..bf1c0a2 --- /dev/null +++ b/src/freshdata/render/options.py @@ -0,0 +1,87 @@ +"""Display configuration for Peel renderers (spec §11). + +Display options never affect cleaning behavior — they are consumed only by +renderers. Precedence: explicit arguments > process-wide :func:`set_display` +> environment (``FRESHDATA_DISPLAY``, ``NO_COLOR``, ``FRESHDATA_NO_PREVIEWS``) +> defaults. +""" + +from __future__ import annotations + +import os +import sys +from dataclasses import dataclass, replace + +#: Valid display modes (spec §11.1). +MODES = ("auto", "compact", "standard", "verbose", "debug", "json", "plain", "silent") + +#: Valid notebook-HTML styles: the legacy layout or the Peel card (spec §17). +STYLES = ("legacy", "peel") + + +@dataclass(frozen=True) +class RenderOptions: + """Resolved display options handed to a renderer.""" + + mode: str = "auto" + color: str = "auto" # "auto" | "always" | "never" + width: int = 74 + previews: bool = True # False → schema-only display, values never shown + ascii_icons: bool = False + style: str = "legacy" # notebook HTML style; "peel" opts into the new card + + def resolved_mode(self, *, isatty: bool | None = None) -> str: + """Collapse ``auto`` to a concrete mode for the current environment.""" + if self.mode != "auto": + return self.mode + if isatty is None: + isatty = sys.stdout.isatty() + return "standard" if isatty else "compact" + + +_state = {"options": RenderOptions()} + + +def set_display(style: str | None = None, **changes: object) -> RenderOptions: + """Set process-wide display preferences. + + ``fd.set_display("peel")`` opts into the Peel notebook style; + ``fd.set_display(mode="compact")`` etc. set individual fields. Returns the + new options. Unknown fields raise ``TypeError``; an unknown mode or style + raises ``ValueError`` so typos fail at configuration time, not render time. + """ + if style is not None: + changes["style"] = style + new = replace(_state["options"], **changes) # type: ignore[arg-type] + if new.mode not in MODES: + raise ValueError(f"unknown display mode {new.mode!r}; expected one of {MODES}") + if new.style not in STYLES: + raise ValueError(f"unknown display style {new.style!r}; expected one of {STYLES}") + _state["options"] = new + return new + + +def get_display(**overrides: object) -> RenderOptions: + """The effective options: defaults ← env ← :func:`set_display` ← *overrides*.""" + opts = _state["options"] + env_display = os.environ.get("FRESHDATA_DISPLAY") + if env_display == "peel" and opts.style == "legacy": + opts = replace(opts, style="peel") + elif env_display and opts.mode == "auto" and env_display in MODES: + opts = replace(opts, mode=env_display) + if os.environ.get("NO_COLOR"): + opts = replace(opts, color="never") + if os.environ.get("FRESHDATA_NO_PREVIEWS"): + opts = replace(opts, previews=False) + if overrides: + opts = replace(opts, **overrides) # type: ignore[arg-type] + if opts.mode not in MODES: + raise ValueError(f"unknown display mode {opts.mode!r}; expected one of {MODES}") + if opts.mode == "plain": + opts = replace(opts, color="never", ascii_icons=True) + return opts + + +def reset_display() -> None: + """Restore defaults (used by tests and ``FRESHDATA_LEGACY_DISPLAY`` flows).""" + _state["options"] = RenderOptions() diff --git a/src/freshdata/render/plain.py b/src/freshdata/render/plain.py new file mode 100644 index 0000000..c22df36 --- /dev/null +++ b/src/freshdata/render/plain.py @@ -0,0 +1,148 @@ +"""Plain-text Peel renderer — no ANSI, works everywhere (spec §11). + +This is the reference renderer: the rich terminal renderer and the notebook +renderer must present the same information; only styling may differ. Because +every state is a text label, output stays grep-able in logs and CI +(``grep REVIEW``, ``grep PARTIAL``). +""" + +from __future__ import annotations + +import json + +from .options import RenderOptions, get_display +from .view import PeelView, Section + +#: view.kind → command-style title. +_TITLES = { + "clean_report": "freshdata clean", + "profile": "freshdata profile", + "parse": "freshdata parse", + "copilot": "freshdata ai-copilot", +} + +_SEVERITY_LABELS = { + "error": "Error", + "warning": "Warning", + "review": "Review", + "info": "Info", +} + + +def _title(view: PeelView) -> str: + return _TITLES.get(view.kind, f"freshdata {view.kind}") + + +def _sep(options: RenderOptions) -> str: + return " - " if options.ascii_icons else " · " + + +def _status_text(view: PeelView, options: RenderOptions) -> str: + return (" - " if options.ascii_icons else " · ").join(view.status) + + +def _rule(text_left: str, text_right: str, options: RenderOptions) -> str: + dash = "-" if options.ascii_icons else "─" + body = f"{dash * 2} {text_left} " + right = f" {text_right} {dash * 2}" if text_right else dash * 2 + fill = max(1, options.width - len(body) - len(right)) + return body + dash * fill + right + + +def _attention_lines(view: PeelView, options: RenderOptions) -> list[str]: + if not view.attention: + return [" nothing needs review"] + lines = [f" Needs attention ({len(view.attention)})"] + subject_width = min(12, max((len(a.subject) for a in view.attention), default=0)) + for item in view.attention: + subject = item.subject[:subject_width].ljust(subject_width) + label = _SEVERITY_LABELS.get(item.severity, item.severity).ljust(8) + lines.append(f" {label} {subject} {item.text} [{item.id}]".rstrip()) + return lines + + +def _section_lines(section: Section) -> list[str]: + lines = [f" {section.title} ({section.count})"] + for row in section.rows(): + cells = ", ".join(f"{k}={_cell(v)}" for k, v in row.items() if v not in ("", None, [])) + lines.append(f" - {cells}") + return lines + + +def _cell(value: object) -> str: + if isinstance(value, float): + return f"{value:g}" + if isinstance(value, (dict, list)): + return json.dumps(value, default=str) + return str(value) + + +def render_plain(view: PeelView, options: RenderOptions | None = None) -> str: + """Render *view* as plain text in the mode carried by *options*.""" + options = options or get_display() + mode = options.resolved_mode() + + if mode == "silent": + return "" + if mode == "json": + report = view.audit_ref + if report is not None and hasattr(report, "to_dict"): + return json.dumps(report.to_dict(), indent=2, default=str) + return json.dumps({"kind": view.kind, "status": list(view.status)}, indent=2) + + sep = _sep(options) + + if mode == "compact": + pieces = [ + _title(view), + _status_text(view, options), + view.headline, + ] + if view.attention: + ids = " ".join(a.id for a in view.attention[:3]) + pieces.append(f"{len(view.attention)} attention ({ids})") + first = " ".join(pieces) + return first + "\n -> report.show() for details" + + # standard / verbose / debug / plain share the panel layout + lines = [_rule(_title(view), _status_text(view, options), options)] + lines.append(f" {view.headline}") + if view.metrics: + chips = [] + for metric in view.metrics: + if metric.before is not None: + arrow = "->" if options.ascii_icons else "→" + chips.append(f"{metric.label} {metric.before}{arrow}{metric.after}") + else: + chips.append(f"{metric.label} {metric.value}") + lines.append(" " + sep.join(chips).strip()) + if view.banner: + lines.append("") + lines.append(f" !! {view.banner}") + lines.append("") + lines.extend(_attention_lines(view, options)) + lines.append("") + if view.next_step: + lines.append(f" next {view.next_step}") + lines.append(f" more report.show(mode=\"verbose\"){sep}report.to_json()") + + if mode in ("verbose", "debug"): + for section in view.sections: + if section.key == "audit" and mode != "debug": + continue + lines.append("") + lines.extend(_section_lines(section)) + + dash = "-" if options.ascii_icons else "─" + lines.append(dash * options.width) + out = "\n".join(lines) + if options.ascii_icons: + out = _asciify(out) + return out + + +def _asciify(text: str) -> str: + """Replace Peel's framing glyphs with ASCII (plain mode / ASCII icon set).""" + for glyph, ascii_form in (("·", "-"), ("→", "->"), ("—", "--"), ("─", "-")): + text = text.replace(glyph, ascii_form) + return text diff --git a/src/freshdata/render/terminal.py b/src/freshdata/render/terminal.py new file mode 100644 index 0000000..6963fe9 --- /dev/null +++ b/src/freshdata/render/terminal.py @@ -0,0 +1,142 @@ +"""Styled terminal renderer over :class:`PeelView`, using ``rich`` when present. + +``rich`` is an optional extra (``pip install freshdata-cleaner[rich]``). Without it — +or for the text-first modes (``compact``/``json``/``plain``/``silent``) — this +module delegates to the plain renderer, which is the reference for content: +the styled output may only differ in styling, never in information. +""" + +from __future__ import annotations + +from typing import Any + +from .options import RenderOptions, get_display +from .plain import render_plain +from .view import PeelView + +_SEVERITY_STYLES = { + "error": "bold red", + "warning": "yellow", + "review": "cyan", + "info": "dim", +} + +_TITLES_STYLE = "bold" + + +def render_terminal_text(view: PeelView, options: RenderOptions | None = None) -> str: + """Render *view* for a terminal, returning the final text (ANSI included + only when color is enabled). Falls back to plain text when ``rich`` is not + installed.""" + options = options or get_display() + mode = options.resolved_mode() + if mode in ("compact", "json", "plain", "silent"): + return render_plain(view, options) + try: + return _render_rich(view, options, mode) + except ImportError: + return render_plain(view, options) + + +def _render_rich(view: PeelView, options: RenderOptions, mode: str) -> str: + from rich.console import Console, Group + from rich.panel import Panel + from rich.table import Table + from rich.text import Text + + color = options.color != "never" + console = Console( + record=True, + width=options.width, + force_terminal=color, + no_color=not color, + highlight=False, + ) + + body: list[Any] = [Text(view.headline)] + if view.metrics: + chips = [] + for metric in view.metrics: + if metric.before is not None: + chips.append(f"{metric.label} {metric.before}→{metric.after}") + else: + chips.append(f"{metric.label} {metric.value}") + body.append(Text(" · ".join(chips), style="dim")) + if view.banner: + body.append(Text(f"!! {view.banner}", style="bold yellow")) + + body.append(Text()) + if view.attention: + table = Table( + title=f"Needs attention ({len(view.attention)})", + title_justify="left", + show_header=False, + box=None, + pad_edge=False, + ) + table.add_column("severity", style="bold") + table.add_column("subject") + table.add_column("text", overflow="fold") + table.add_column("id", style="dim") + for item in view.attention: + table.add_row( + Text(item.severity.capitalize(), style=_SEVERITY_STYLES.get(item.severity, "")), + item.subject, + item.text, + f"[{item.id}]", + ) + body.append(table) + else: + body.append(Text("nothing needs review", style="dim")) + + tail: list[str] = [] + if view.next_step: + tail.append(f"next {view.next_step}") + tail.append('more report.show(mode="verbose") · report.to_json()') + body.append(Text()) + body.append(Text("\n".join(tail), style="dim")) + + if mode in ("verbose", "debug"): + for section in view.sections: + if section.key == "audit" and mode != "debug": + continue + table = Table( + title=f"{section.title} ({section.count})", title_justify="left", box=None + ) + rows = section.rows() + columns: list[str] = [] + for row in rows: + for key in row: + if key not in columns: + columns.append(key) + for key in columns: + table.add_column(key) + for row in rows: + table.add_row(*(_cell(row.get(key)) for key in columns)) + body.append(Text()) + body.append(table) + + console.print( + Panel( + Group(*body), + title=Text(_panel_title(view), style=_TITLES_STYLE), + subtitle=view.status_label, + title_align="left", + subtitle_align="right", + ) + ) + return console.export_text(styles=color) + + +def _panel_title(view: PeelView) -> str: + from .plain import _TITLES + + return _TITLES.get(view.kind, f"freshdata {view.kind}") + + +def _cell(value: object) -> str: + if value in (None, "", [], {}): + return "" + if isinstance(value, float): + return f"{value:g}" + return str(value) diff --git a/src/freshdata/render/view.py b/src/freshdata/render/view.py new file mode 100644 index 0000000..49cfdab --- /dev/null +++ b/src/freshdata/render/view.py @@ -0,0 +1,134 @@ +"""Peel view model — the display-neutral middle layer of freshdata's output system. + +Every report object is *normalized* (see :mod:`freshdata.render.normalize`) into a +:class:`PeelView`, and every renderer (plain text, rich terminal, notebook HTML) +consumes only views. Normalization is pure and deterministic: the same report +always produces the same view, so renderers never re-derive or re-rank anything. + +The vocabulary here is the single source of truth for Peel's shared grammar: + +* result **statuses** (``CLEAN``/``CHANGED``/``REVIEW``/``BLOCKED``/``PARTIAL``/ + ``SKIPPED``/``FAILED``) — always rendered as text labels, never color alone; +* finding **severities** (``error``/``warning``/``review``/``info``) with one + shared ranking used by every attention list; +* the **confidence ladder** mapping scores to plain-language phrases. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Callable + +#: Result statuses, in display order. A view may carry several (e.g. CHANGED+REVIEW). +STATUSES = ("CLEAN", "CHANGED", "REVIEW", "BLOCKED", "PARTIAL", "SKIPPED", "FAILED") + +#: Finding severities, most urgent first. +SEVERITIES = ("error", "warning", "review", "info") + +#: Attention-domain ranks (spec §5.3): privacy/safety outrank everything, +#: cosmetic consistency comes last. Normalizers tag each item with one. +DOMAINS = ("privacy", "corruption", "policy", "reliability", "cosmetic") + +_SEVERITY_RANK = {name: i for i, name in enumerate(SEVERITIES)} +_DOMAIN_RANK = {name: i for i, name in enumerate(DOMAINS)} + + +def confidence_phrase(confidence: float, *, ambiguous: bool = False) -> str: + """The plain-language rung of the confidence ladder for *confidence*. + + ``ambiguous=True`` forces the bottom rung regardless of score (a near-tie + between candidates is ambiguous even when the winner scored well). + """ + if ambiguous or confidence < 0.60: + return "ambiguous — no change made" if ambiguous else "uncertain" + if confidence >= 0.95: + return "strong evidence" + if confidence >= 0.80: + return "moderate evidence" + return "uncertain" + + +@dataclass(frozen=True) +class Metric: + """One glance-layer number, optionally as a before/after pair.""" + + label: str + value: str + before: str | None = None + after: str | None = None + + +@dataclass(frozen=True) +class AttentionItem: + """One ranked finding in the attention list. + + ``id`` is stable within a run (``W1``, ``R2``, ``D1``, ...) so terminal + output, notebook output, and programmatic access all name the same thing. + """ + + id: str + severity: str # one of SEVERITIES + subject: str # column / frame / finding target ("" for table-level) + text: str # plain-language, one sentence + domain: str = "reliability" # one of DOMAINS + count: int = 0 # affected cells/rows (0 = not applicable) + detail: dict[str, Any] = field(default_factory=dict) # audit-layer payload + + def sort_key(self) -> tuple[int, int, int, str, str, str]: + """Shared comparator (spec §5.3): domain, severity, -count, name order, + then the stable id so ranking is total (independent of input order).""" + return ( + _DOMAIN_RANK.get(self.domain, len(DOMAINS)), + _SEVERITY_RANK.get(self.severity, len(SEVERITIES)), + -self.count, + self.subject, + self.text, + self.id, + ) + + +@dataclass(frozen=True) +class Section: + """One inspect-layer group. ``body`` is a thunk so detail costs nothing + until a renderer actually expands it (spec §16).""" + + key: str # machine name, e.g. "columns", "actions", "audit" + title: str # human name, e.g. "Column changes" + body: Callable[[], list[dict[str, Any]]] # lazy rows, JSON-friendly + count: int = 0 # advertised size without evaluating body + + def rows(self) -> list[dict[str, Any]]: + return self.body() + + +@dataclass(frozen=True) +class PeelView: + """The display-neutral shape of one report (spec §12.2). + + ``audit_ref`` is the original report object — layer 3 is never a rendering. + """ + + kind: str # "clean_report", "parse", "copilot", ... + status: tuple[str, ...] # subset of STATUSES, display order + headline: str # one-sentence impact + metrics: tuple[Metric, ...] + attention: tuple[AttentionItem, ...] # already ranked + next_step: str | None # one runnable snippet, or None + sections: tuple[Section, ...] + audit_ref: Any = None + banner: str | None = None # PARTIAL/experimental banner text, if any + + def __post_init__(self) -> None: + unknown = [s for s in self.status if s not in STATUSES] + if unknown: + raise ValueError(f"unknown status labels: {unknown}") + + @property + def status_label(self) -> str: + """The combined text label, e.g. ``"CHANGED · REVIEW"``.""" + return " · ".join(self.status) + + +def rank_attention(items: list[AttentionItem]) -> tuple[AttentionItem, ...]: + """Sort *items* by the shared comparator. Deterministic; stable for ties.""" + return tuple(sorted(items, key=AttentionItem.sort_key)) diff --git a/tests/test_peel_cli.py b/tests/test_peel_cli.py new file mode 100644 index 0000000..20b4fdb --- /dev/null +++ b/tests/test_peel_cli.py @@ -0,0 +1,76 @@ +"""Layer 10: additive Peel display flags on `freshdata clean`.""" + +from __future__ import annotations + +import json + +import pytest + +from freshdata.enterprise import cli + + +@pytest.fixture +def csv(tmp_path): + path = tmp_path / "in.csv" + path.write_text("a,b\n1,x\n1,x\n,y\n", encoding="utf-8") + return str(path) + + +def run(argv, capsys): + code = cli.main(argv) + return code, capsys.readouterr().out + + +class TestBackwardCompatibleDefault: + def test_default_output_is_legacy_summary(self, csv, capsys): + code, out = run(["clean", csv], capsys) + assert code == 0 + # legacy enterprise engine summary wording, unchanged + assert "freshdata enterprise" in out + assert "trust" in out + assert "╭─" not in out # no Peel panel by default + + def test_quiet_still_suppresses(self, csv, capsys): + _, out = run(["clean", csv, "--quiet"], capsys) + assert "freshdata enterprise" not in out + + +class TestDisplayFlags: + def test_output_format_json_emits_report_dict(self, csv, capsys): + code, out = run(["clean", csv, "--output-format", "json"], capsys) + assert code == 0 + payload = json.loads(out) + assert payload["rows_before"] == 3 + assert payload["rows_after"] == 2 + + def test_verbose_renders_peel(self, csv, capsys): + _, out = run(["clean", csv, "--verbose", "--no-color"], capsys) + assert "freshdata clean" in out + assert "CHANGED" in out or "CLEAN" in out + assert "\x1b[" not in out # --no-color strips ANSI + + def test_vv_is_debug_mode(self, csv, capsys): + _, out = run(["clean", csv, "-vv", "--no-color"], capsys) + # debug mode surfaces the audit section + assert "Audit" in out + + def test_display_peel_without_verbose(self, csv, capsys): + _, out = run(["clean", csv, "--display", "peel", "--no-color"], capsys) + assert "freshdata clean" in out + assert "freshdata clean report" not in out # not the legacy text + + def test_report_file_flag_unaffected(self, csv, tmp_path, capsys): + # --report still writes the existing enterprise wrapper (clean_report nested) + report_path = tmp_path / "r.json" + run(["clean", csv, "--report", str(report_path)], capsys) + payload = json.loads(report_path.read_text()) + assert payload["clean_report"]["rows_before"] == 3 + + def test_json_stdout_and_report_file_coexist(self, csv, tmp_path, capsys): + report_path = tmp_path / "r.json" + _, out = run( + ["clean", csv, "--report", str(report_path), "--output-format", "json"], capsys + ) + # stdout json is the inner CleanReport; the file keeps the wrapper + assert json.loads(out)["rows_before"] == 3 + assert json.loads(report_path.read_text())["clean_report"]["rows_before"] == 3 diff --git a/tests/test_peel_copilot.py b/tests/test_peel_copilot.py new file mode 100644 index 0000000..c5479a6 --- /dev/null +++ b/tests/test_peel_copilot.py @@ -0,0 +1,158 @@ +"""Layer 9: CopilotReport display and the privacy-over-trust invariant (spec §8).""" + +from __future__ import annotations + +import pytest + +from freshdata.enterprise.metrics import TrustScore +from freshdata.experimental.ai_copilot import ( + CleaningPlan, + CopilotReport, + DetectedProblem, + PlanStep, +) +from freshdata.render.normalize import normalize +from freshdata.render.options import get_display, reset_display +from freshdata.render.plain import render_plain + + +@pytest.fixture(autouse=True) +def _reset(monkeypatch): + for var in ("FRESHDATA_DISPLAY", "FRESHDATA_LEGACY_DISPLAY", "NO_COLOR"): + monkeypatch.delenv(var, raising=False) + reset_display() + yield + reset_display() + + +def make_report( + *, + pii="2 columns look like personal data (email, phone)", + problems=(("missing", "medium", "income is 38% missing", "income", 380),), + policy_violations=(), + trust_overall=71.0, + grade_dims=(62.0, 78.0, 99.0, 45.0), + provider_error=None, + steps=(("Mask PII", "fd.clean(df, context='mask emails')"),), +) -> CopilotReport: + audit = {"model_context_sha256": "3f9cabc", "pii_columns": ["email", "phone"]} + if provider_error: + audit["provider_error"] = provider_error + completeness, validity, uniqueness, consistency = grade_dims + return CopilotReport( + goal="prep for churn model", + summary="deterministic summary text", + problems=tuple( + DetectedProblem(kind=k, severity=s, detail=d, column=c, count=n) + for k, s, d, c, n in problems + ), + pii_warning=pii, + policy_violations=tuple(policy_violations), + cleaning_plan=CleaningPlan( + steps=tuple( + PlanStep(order=i + 1, action=a, rationale="because", tool=t) + for i, (a, t) in enumerate(steps) + ) + ), + recommended_code="df = fd.clean(df, context='mask emails')", + trust=TrustScore( + overall=trust_overall, + completeness=completeness, + validity=validity, + uniqueness=uniqueness, + consistency=consistency, + n_rows=10_000, + n_cols=14, + ), + model_context={"columns": ["email", "phone"], "sample": "masked"}, + audit=audit, + narrative=None if provider_error else "the data looks...", + ) + + +class TestPrivacyOverTrust: + def test_pii_ranks_first_even_with_grade_a(self): + rep = make_report(trust_overall=95.0, grade_dims=(99, 99, 99, 99)) + view = normalize(rep) + assert view.attention[0].id == "A1" + assert view.attention[0].domain == "privacy" + assert "REVIEW" in view.status # grade A cannot suppress REVIEW + + def test_high_trust_does_not_remove_pii_from_glance(self): + rep = make_report(trust_overall=95.0, grade_dims=(99, 99, 99, 99)) + out = render_plain(normalize(rep), get_display(mode="standard")) + assert "personal data" in out + assert "REVIEW" in out + + def test_policy_violation_outranks_lower_severity_data_quality(self): + # spec §5.3 order: privacy > corruption > policy > reliability > cosmetic, + # so policy outranks a medium/low data-quality issue but not a corruption one + rep = make_report( + pii=None, + problems=(("consistency", "low", "minor casing noise", "tier", 3),), + policy_violations=({"message": "income must be clustered", "column": "income"},), + ) + view = normalize(rep) + assert view.attention[0].domain == "policy" + assert view.attention[-1].domain == "cosmetic" + + +class TestQueue: + def test_ids_assigned_after_ranking(self): + view = normalize(make_report()) + assert [a.id for a in view.attention] == [ + f"A{i}" for i in range(1, len(view.attention) + 1) + ] + + def test_no_pii_is_explicit_not_absent(self): + view = normalize(make_report(pii=None)) + privacy_metric = next(m for m in view.metrics if m.label == "privacy") + assert "no PII detected" in privacy_metric.value + + def test_trust_metric_carries_scope_note(self): + view = normalize(make_report()) + trust_metric = next(m for m in view.metrics if m.label == "trust") + assert "data quality only" in trust_metric.value + assert "71/100 (C)" in trust_metric.value + + +class TestExperimentalAndProvider: + def test_experimental_banner_always_present(self): + assert "Experimental API" in normalize(make_report()).banner + + def test_provider_failure_is_labeled_not_absent(self): + view = normalize(make_report(provider_error="TimeoutError: provider timed out")) + assert "PARTIAL" in view.status + assert "narrative unavailable" in view.banner.lower() + assert "deterministic findings above are complete" in view.banner + + def test_next_step_is_first_plan_tool(self): + view = normalize(make_report()) + assert view.next_step == "fd.clean(df, context='mask emails')" + + +class TestRendering: + def test_attention_property_matches_view(self): + rep = make_report() + assert rep.attention == normalize(rep).attention + + def test_repr_html_renders_peel_for_copilot(self): + # copilot has no legacy renderer → always Peel + html = make_report().to_html() + assert "freshdata ai-copilot" in html + assert "Needs attention" in html + assert "personal data" in html + + def test_str_still_returns_summary_string(self): + # backward-compat: __str__ unchanged + assert str(make_report()) == "deterministic summary text" + + def test_verbose_shows_plan_and_masked_context(self): + out = render_plain(normalize(make_report()), get_display(mode="verbose")) + assert "Cleaning plan" in out + assert "Model context" in out + assert "masked" in out + + def test_recommended_code_flagged_machine_generated(self): + out = render_plain(normalize(make_report()), get_display(mode="verbose")) + assert "machine-generated" in out diff --git a/tests/test_peel_normalize.py b/tests/test_peel_normalize.py new file mode 100644 index 0000000..4dd1c93 --- /dev/null +++ b/tests/test_peel_normalize.py @@ -0,0 +1,258 @@ +"""Layer 2: CleanReport → PeelView normalization.""" + +from __future__ import annotations + +import pytest + +from freshdata.render.normalize import normalize, normalize_clean_report, register_normalizer +from freshdata.render.view import PeelView +from freshdata.report import CleanReport + + +def make_report(**kw) -> CleanReport: + rep = CleanReport( + rows_before=10_000, + rows_after=9_988, + cols_before=14, + cols_after=14, + missing_before=1_204, + missing_after=0, + duration_seconds=0.412, + ) + for key, value in kw.items(): + setattr(rep, key, value) + return rep + + +class TestDispatch: + def test_normalize_dispatches_on_render_kind(self): + view = normalize(make_report()) + assert isinstance(view, PeelView) + assert view.kind == "clean_report" + + def test_unknown_kind_raises(self): + with pytest.raises(KeyError, match="no Peel normalizer"): + normalize(object()) + + def test_register_normalizer_extends_dispatch(self): + class Custom: + _render_kind = "custom_thing" + + view = PeelView("custom_thing", ("CLEAN",), "ok", (), (), None, ()) + register_normalizer("custom_thing", lambda obj: view) + assert normalize(Custom()) is view + + +class TestStatus: + def test_untouched_data_is_clean(self): + assert normalize_clean_report(make_report()).status == ("CLEAN",) + + def test_changes_make_changed(self): + rep = make_report() + rep.add("missing", "filled 214 values", column="age", count=214) + assert normalize_clean_report(rep).status == ("CHANGED",) + + def test_warnings_add_review(self): + rep = make_report(warnings=["column 'income' is 38% missing"]) + assert normalize_clean_report(rep).status == ("CLEAN", "REVIEW") + + def test_unmaterialized_adds_partial_and_banner(self): + rep = make_report(materialized=False) + view = normalize_clean_report(rep) + assert "PARTIAL" in view.status + assert "kept in the engine" in view.banner + assert "kept in the engine" in view.headline + + def test_info_only_attention_is_not_review(self): + rep = make_report( + fallback_events=[ + {"backend": "polars", "fallback_step": "impute", "fallback_reason": "dtype"} + ] + ) + view = normalize_clean_report(rep) + assert "REVIEW" not in view.status + assert view.attention[0].severity == "info" + + +class TestAttention: + def test_ids_are_stable_per_source(self): + rep = make_report( + warnings=["w-one", "w-two"], + recommendations=["r-one"], + ) + view = normalize_clean_report(rep) + assert [a.id for a in view.attention] == ["W1", "W2", "R1"] + assert [a.severity for a in view.attention] == ["warning", "warning", "review"] + + def test_domain_errors_outrank_report_warnings(self): + rep = make_report( + warnings=["some warning"], + domain="healthcare", + domain_findings=[ + { + "status": "violated", + "severity": "error", + "column": "obx_units", + "message": "47 values are not valid UCUM units", + "count": 47, + } + ], + ) + top = normalize_clean_report(rep).attention[0] + assert top.id == "D1" + assert top.severity == "error" + assert top.domain == "policy" + assert top.count == 47 + + def test_passed_domain_findings_are_not_attention(self): + rep = make_report( + domain="energy", + domain_findings=[{"status": "passed", "severity": "error", "rule": "range"}], + ) + assert normalize_clean_report(rep).attention == () + + def test_suggested_actions_become_review_items(self): + rep = make_report() + # non-semantic suggested action → generic "held for your review" text + rep.add( + "impute", + "median fill needs confirmation on skewed column", + column="income", + count=380, + status="suggested", + confidence=0.84, + ) + view = normalize_clean_report(rep) + (item,) = view.attention + assert item.id == "S1" + assert item.severity == "review" + assert item.subject == "income" + assert "held for your review" in item.text + assert item.detail["action"]["confidence"] == 0.84 + + def test_contract_failures_become_policy_items(self): + rep = make_report( + contract_violations={ + "passed": False, + "baseline_name": "orders", + "baseline_version": 3, + "findings": [ + {"status": "failed", "column": "amount", "message": "dtype drift", + "check_id": "dtype"}, + {"status": "passed", "column": "id", "check_id": "presence"}, + ], + } + ) + view = normalize_clean_report(rep) + (item,) = view.attention + assert item.id == "C1" + assert item.severity == "error" + assert item.domain == "policy" + assert "contract 'orders'" in item.text + + def test_fallback_events_are_plain_language_info(self): + rep = make_report( + fallback_events=[ + {"backend": "duckdb", "fallback_step": "outliers", "fallback_reason": "quantile"} + ] + ) + (item,) = normalize_clean_report(rep).attention + assert item.id == "F1" + assert "continued without the optional engine step" in item.text + + def test_deterministic_across_calls(self): + rep = make_report(warnings=["a", "b"], recommendations=["c"]) + first = normalize_clean_report(rep) + second = normalize_clean_report(rep) + assert first.attention == second.attention + assert first.status == second.status + assert first.headline == second.headline + + +class TestHeadlineAndMetrics: + def test_headline_contents(self): + rep = make_report() + rep.add("missing", "filled", column="age", count=214) + headline = normalize_clean_report(rep).headline + assert "9,988 of 10,000 rows kept" in headline + assert "14 columns" in headline + assert "214 cells changed" in headline + assert "0.4s" in headline + + def test_metrics_include_missing_and_protected(self): + rep = make_report(duplicates_removed=12, columns_preserved=["notes"]) + labels = {m.label: m for m in normalize_clean_report(rep).metrics} + assert labels["missing"].before == "1,204" + assert labels["missing"].after == "0" + assert labels["duplicates"].value == "-12" + assert labels["protected"].value == "1" + + +class TestNextStep: + def test_no_attention_no_next_step(self): + assert normalize_clean_report(make_report()).next_step is None + + def test_info_only_no_next_step(self): + rep = make_report( + fallback_events=[{"backend": "polars", "fallback_step": "x", "fallback_reason": "y"}] + ) + assert normalize_clean_report(rep).next_step is None + + def test_warning_points_to_explain(self): + rep = make_report(warnings=["income is 38% missing"]) + assert "explain_clean" in normalize_clean_report(rep).next_step + + def test_suggested_actions_point_to_plan(self): + rep = make_report() + rep.add("semantic", "proposed fix", column="c", count=2, status="suggested") + assert "suggest_plan" in normalize_clean_report(rep).next_step + + def test_domain_findings_win_over_warnings(self): + rep = make_report( + warnings=["w"], + domain="finance", + domain_findings=[ + {"status": "violated", "severity": "error", "column": "px", "message": "bad"} + ], + ) + assert "domain_findings" in normalize_clean_report(rep).next_step + + +class TestSections: + def test_column_rows_aggregate_and_order(self): + rep = make_report(columns_preserved=["notes"]) + rep.add("missing", "filled 214 value(s) with median", column="age", count=214) + rep.add("outliers", "winsorized 37 value(s)", column="income", count=37) + rep.add("missing", "filled 380 value(s) with median", column="income", count=380) + view = normalize_clean_report(rep) + columns = next(s for s in view.sections if s.key == "columns") + rows = columns.rows() + assert rows[0]["column"] == "income" + assert rows[0]["changed"] == 417 + assert rows[0]["risk"] == "low" + assert "filled missing values" in rows[0]["what"] + assert "extreme values adjusted" in rows[0]["what"] + protected = next(r for r in rows if r["column"] == "notes") + assert protected["what"] == "protected — left untouched" + + def test_actions_section_uses_report_schema(self): + rep = make_report() + rep.add("missing", "filled", column="age", count=214, rationale="median safe") + actions = next(s for s in normalize_clean_report(rep).sections if s.key == "actions") + assert actions.count == 1 + (row,) = actions.rows() + assert row["step"] == "missing" + assert row["rationale"] == "median safe" + + def test_audit_section_skips_empty_and_keeps_hash(self): + rep = make_report(decisions_hash="abc123", backend="polars") + audit = next(s for s in normalize_clean_report(rep).sections if s.key == "audit") + fields = {r["field"]: r["value"] for r in audit.rows()} + assert fields["decisions_hash"] == "abc123" + assert fields["backend"] == "polars" + assert "streaming" not in fields + assert "undo_log" not in fields # never serialized (spec §13) + + def test_audit_ref_is_the_report(self): + rep = make_report() + assert normalize_clean_report(rep).audit_ref is rep diff --git a/tests/test_peel_notebook.py b/tests/test_peel_notebook.py new file mode 100644 index 0000000..5607c8a --- /dev/null +++ b/tests/test_peel_notebook.py @@ -0,0 +1,126 @@ +"""Layer 6: the Peel notebook HTML renderer and its opt-in gate.""" + +from __future__ import annotations + +import pytest + +import freshdata as fd +from freshdata.render.normalize import normalize_clean_report +from freshdata.render.notebook import render_notebook +from freshdata.render.options import reset_display +from freshdata.report import CleanReport + + +@pytest.fixture(autouse=True) +def _clean_display_state(monkeypatch): + for var in ( + "FRESHDATA_DISPLAY", + "FRESHDATA_LEGACY_DISPLAY", + "NO_COLOR", + "FRESHDATA_NO_PREVIEWS", + ): + monkeypatch.delenv(var, raising=False) + reset_display() + yield + reset_display() + + +def make_report(**kw) -> CleanReport: + rep = CleanReport( + rows_before=1_000, rows_after=998, cols_before=5, cols_after=5, + missing_before=40, missing_after=0, duration_seconds=0.2, + ) + for key, value in kw.items(): + setattr(rep, key, value) + rep.add("missing", "filled 40 value(s) with median", column="age", count=40) + return rep + + +def make_view(**kw): + return normalize_clean_report(make_report(**kw)) + + +class TestPeelHtml: + def test_contains_status_headline_attention(self): + html = render_notebook(make_view(warnings=["column 'x' is mostly empty"])) + assert "freshdata clean" in html + assert "CHANGED" in html and "REVIEW" in html + assert "998 of 1,000 rows kept" in html + assert "Needs attention (1)" in html + assert "column 'x' is mostly empty" in html or "column 'x'" in html + assert "[W1]" in html + + def test_empty_attention_is_explicit(self): + assert "nothing needs review" in render_notebook(make_view()) + + def test_sections_are_collapsed_details(self): + html = render_notebook(make_view()) + assert "
    " in html + assert "Column changes" in html + assert "All actions" in html + # collapsed by default: no open attribute + assert "
    " not in html + + def test_banner_present_for_partial(self): + html = render_notebook(make_view(materialized=False)) + assert "PARTIAL" in html + assert "kept in the engine" in html + + def test_next_step_rendered_as_code(self): + html = render_notebook(make_view(warnings=["w"])) + assert "" in html and "explain_clean" in html + + def test_json_export_control_present(self): + assert "JSON" in render_notebook(make_view()) + + def test_hostile_column_name_is_escaped(self): + rep = make_report() + rep.add_warning(' in column ""') + html = render_notebook(normalize_clean_report(rep)) + assert "