From a01c08d3acf2f95a6eeddb38d4db3e687b600db3 Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Mon, 12 Oct 2026 16:56:36 -0700 Subject: [PATCH 01/25] Enable gradient explanations for kumo-tabular and kumo-relational Allow the explanation output for kumo-tabular (one scored row), return it on result.explanation, and drop the old 1,000-row context rule for relational explanations, which the SDM NIM does not have. Signed-off-by: Jana Gagacheva --- CHANGELOG.md | 6 + docs/reference/limits.md | 2 +- .../src/sdm_client/adapters/relational.py | 51 +------- .../src/sdm_client/adapters/tabular.py | 30 ++++- .../src/sdm_client/models.py | 5 +- .../tests/conftest.py | 43 +++++- .../tests/test_limits_and_absent_outputs.py | 27 ++-- .../tests/test_limits_and_sessions.py | 22 ++-- .../tests/test_request_and_dispatch.py | 12 +- .../tests/test_tabular_explanations.py | 123 ++++++++++++++++++ qa/probe_tabular.py | 6 +- skills/relational/SKILL.md | 3 +- skills/tabular/SKILL.md | 12 +- 13 files changed, 252 insertions(+), 90 deletions(-) create mode 100644 packages/structured-data-models-client/tests/test_tabular_explanations.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 156de1221..0606536fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,9 @@ these packages. All five distributions share one version and one entry. installs on a laptop rather than on Linux only. The engine ships no source distribution, so those wheels remain the full set of platforms it supports: `structured-data-models-client` itself is pure Python and stays broader. +- `kumo-tabular` accepts `explanation` in `outputs`, over one scored row at a + time, and returns it on `result.explanation`. Asking for one over more rows + is refused before sending. ### Changed @@ -42,6 +45,9 @@ these packages. All five distributions share one version and one entry. value that cannot be read as its type, such as a `run_mode` that is not text, is refused with `INVALID_REQUEST` before sending, where a NIM answered `422 VALIDATION_FAILED`. +- A `kumo-relational` explanation is no longer refused above the 1,000-row + context budget. That limit belonged to an earlier deployment; the Kumo + Relational SDM NIM explains under either budget. ### Fixed diff --git a/docs/reference/limits.md b/docs/reference/limits.md index 1700037b3..c6eae68ba 100644 --- a/docs/reference/limits.md +++ b/docs/reference/limits.md @@ -28,7 +28,7 @@ Both endpoints enforce sizes, and most of these are the relational one's. | request body | relational | 30 MiB per call | at `materialize()` | | request body | relational, through Databricks model serving | 16 MB per call, the gateway's own decimal limit | before sending, on the wrapped envelope | | an explanation | relational | exactly one entity | before sending | -| an explanation | relational | only within the 1,000-row context budget | before sending | +| an explanation | tabular | exactly one row | before sending | | hops | relational | 6 | at `materialize()` | | tables | relational | 15 | against the graph the request carries, not the schema it was built from | diff --git a/packages/structured-data-models-client/src/sdm_client/adapters/relational.py b/packages/structured-data-models-client/src/sdm_client/adapters/relational.py index dbd8e9816..3691cbcc4 100644 --- a/packages/structured-data-models-client/src/sdm_client/adapters/relational.py +++ b/packages/structured-data-models-client/src/sdm_client/adapters/relational.py @@ -365,8 +365,7 @@ def _preflight( """Refuse what the endpoint enforces, before spending the round trip. An explanation is the narrow case: the model explains one entity at a - time, and only in the mode the smaller context budget selects. Sending - more gets a 500 that names neither limit, so both are checked here. + time, and the endpoint refuses more, so that limit is checked here. `sends` names the halves this call puts on the wire. A session pins its context at `fit()` and a scoring call carries only rows, so judging the @@ -398,19 +397,6 @@ def _preflight( details={'model': self.spec.model_id, 'scored': scored}, ) - cap = limits['context_rows_fast'] - if 'context' in sends and _asks_for_the_larger_context(payloads, cap): - raise UnsupportedRequestError( - f'an explanation is only produced in the mode the smaller ' - f'context budget selects, and this request was gathered with ' - f'num_context_samples above {cap:,}. Re-materialize with ' - f'num_context_samples={cap} or fewer', - details={ - 'model': self.spec.model_id, - 'context_rows_fast': cap, - }, - ) - def _check_class_count( self, payloads: tuple[dict[str, Any], ...], limits: Mapping[str, int] ) -> None: @@ -632,41 +618,6 @@ def _columns_over(payload: Mapping[str, Any], limit: int) -> list[str]: ] -def _asks_for_the_larger_context( - payloads: tuple[dict[str, Any], ...], cap: int -) -> bool: - r"""Whether this request selected the mode that will not explain. - - The endpoint gates explaining on ``inference.run_mode``, which the client - derives from the *requested* ``num_context_samples``. Counting the gathered - rows instead misses every graph that supplies less context than was asked - for: 120 rows gathered under a request for 1,500 passed this guard and the - endpoint's refusal escaped, naming a knob the caller never set. - - A payload the engine did not gather carries no ``run_mode``, and for that - one the rows are all there is to read. - """ - modes = { - str(payload['inference']['run_mode']) - for payload in payloads - if isinstance(payload.get('inference'), Mapping) - and payload['inference'].get('run_mode') is not None - } - if modes: - return modes != {'fast'} - return ( - sum( - len( - payload.get('context', {}) - .get('instance_table', {}) - .get('rows', []) - ) - for payload in payloads - ) - > cap - ) - - _WIRE_TARGET = 'TARGET' _A_PREDICTED_COLUMN = re.compile( r'^\s*PREDICT\s+([A-Za-z_]\w*\.[A-Za-z_]\w*)\s', re.IGNORECASE diff --git a/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py b/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py index 240485a33..5b4ce5b4d 100644 --- a/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py +++ b/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py @@ -92,7 +92,9 @@ def prediction_plan( have to carry the same context, so splitting would multiply the upload without raising the cap. """ - yield self._envelope(request, outputs) + envelope = self._envelope(request, outputs) + self._check_explanation_rows(request, outputs) + yield envelope def session_plan( self, request: Request, outputs: tuple[str, ...] @@ -123,6 +125,7 @@ def session_prediction_plan( raise _a_row_the_pinned_schema_cannot_hold( request, declared, error ) from error + self._check_explanation_rows(request, outputs) yield { key: envelope[key] for key in _SESSION_SCORING if key in envelope } @@ -163,12 +166,16 @@ def answered( request_id: str = '', metadata: Mapping[str, Any] | None = None, ) -> PredictionResult: + explanation = next( + (item.explanation for item in items if item.explanation), None + ) return PredictionResult( model=model, request_id=request_id, items=as_declared_target(_named_entities(items, payload), payload), metadata=dict(metadata or {}), outputs=outputs, + explanation=explanation, ) def explain_error( @@ -182,6 +189,27 @@ def close(self) -> None: r"""Nothing to release: this adapter sends through the client's own transport and opens nothing of its own.""" + def _check_explanation_rows( + self, request: Request, outputs: tuple[str, ...] + ) -> None: + r"""Refuse an explanation over more rows than the endpoint explains. + + The endpoint explains one row at a time and refuses more with + ``UNSUPPORTED_OUTPUT``, so the limit is named here before anything is + sent. Only the rows being scored count: a session's pinned context is + never judged by it. + """ + if 'explanation' not in outputs: + return + allowed = self.spec.runtime.limits['explanation_predict_rows'] + scored = len(request.predict.instance_table) + if scored != allowed: + raise UnsupportedRequestError( + f'an explanation covers exactly {allowed} row, and this ' + f'request scores {scored}; ask for one at a time', + details={'model': self.spec.model_id, 'scored': scored}, + ) + def _envelope( self, request: Request, diff --git a/packages/structured-data-models-client/src/sdm_client/models.py b/packages/structured-data-models-client/src/sdm_client/models.py index 547e90cca..3e0a88226 100644 --- a/packages/structured-data-models-client/src/sdm_client/models.py +++ b/packages/structured-data-models-client/src/sdm_client/models.py @@ -89,6 +89,7 @@ # large context to be told the limit at the far end. TABULAR_LIMITS = { 'context_rows_max': 10_000, + 'explanation_predict_rows': 1, } KUMO_TABULAR = ModelSpec( @@ -98,7 +99,9 @@ # The endpoint takes the coarse kind only and decides binary versus # multiclass itself, so this client never narrows it here. tasks=frozenset({'classification', 'regression'}), - outputs=frozenset({'prediction', 'probabilities', 'quantiles'}), + outputs=frozenset( + {'prediction', 'probabilities', 'quantiles', 'explanation'} + ), limits=TABULAR_LIMITS, ), ) diff --git a/packages/structured-data-models-client/tests/conftest.py b/packages/structured-data-models-client/tests/conftest.py index 522a18b9e..652c4b081 100644 --- a/packages/structured-data-models-client/tests/conftest.py +++ b/packages/structured-data-models-client/tests/conftest.py @@ -2,13 +2,21 @@ # All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Small random-weight models, standing in for the published ones so that -nothing is downloaded and no GPU is needed.""" +"""Fixtures shared across the client's test modules. + +The models are small random-weight ones, standing in for the published ones so +that nothing is downloaded and no GPU is needed. +""" from __future__ import annotations +from collections.abc import Callable + +import pandas as pd import pytest +from sdm_client import Request + @pytest.fixture(scope='module') def sdm(): @@ -30,3 +38,34 @@ def relational(sdm): torch.manual_seed(0) return sdm.models.KumoRelational(pretrained=False, device='cpu') + + +@pytest.fixture +def tabular_request() -> Callable[..., Request]: + """Build a single-table request: 20 labelled context rows, ``rows`` to score. + + A classification target alternates 0/1; a regression target is a float. + """ + + def build(rows: int = 1, task: str = 'classification') -> Request: + labels = [ + i % 2 if task == 'classification' else float(i) for i in range(20) + ] + context = pd.DataFrame( + { + 'row_id': range(20), + 'age': [20.0 + i for i in range(20)], + 'churn': labels, + } + ) + predict = pd.DataFrame( + { + 'row_id': range(90, 90 + rows), + 'age': [33.0 + i for i in range(rows)], + } + ) + return Request( + context=context, predict=predict, target='churn', task=task + ) + + return build diff --git a/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py b/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py index 6fba94865..f4ce97362 100644 --- a/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py +++ b/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py @@ -324,24 +324,29 @@ def explanation_plan(request): ) -def test_an_explanation_is_refused_on_the_mode_the_endpoint_refuses_it_on(): - """A graph that supplies 120 rows for a request of 1,500 still selects the - larger-context mode, and that is what the endpoint gates explaining on.""" - with pytest.raises(StructuredDataError) as raised: - explanation_plan(gathered(context_rows=120, run_mode='best')) - assert 'num_context_samples' in str(raised.value) +def test_explains_larger_context(): + """An explanation is sent under the larger-context run mode. + + Assumptions: + - The Kumo Relational SDM NIM explains under either context budget. + - The payload scores one entity. + """ + assert explanation_plan(gathered(context_rows=120, run_mode='best')) def test_a_small_context_asked_for_smally_still_explains(): assert explanation_plan(gathered(context_rows=120, run_mode='fast')) -def test_a_payload_naming_no_mode_is_judged_on_the_rows_it_carries(): - """Nothing the engine gathers arrives without a run_mode, but a hand-built - payload can, and the rows are then all there is to read.""" +def test_explains_without_mode(): + """An explanation is sent whatever the context size when no run mode is + named. + + Assumptions: + - The payload scores one entity and names no run_mode. + """ assert explanation_plan(gathered(context_rows=120, run_mode=None)) - with pytest.raises(StructuredDataError): - explanation_plan(gathered(context_rows=1_001, run_mode=None)) + assert explanation_plan(gathered(context_rows=1_001, run_mode=None)) # --- a context that teaches nothing, on both routes ------------------------- diff --git a/packages/structured-data-models-client/tests/test_limits_and_sessions.py b/packages/structured-data-models-client/tests/test_limits_and_sessions.py index 462d90665..b1459e17d 100644 --- a/packages/structured-data-models-client/tests/test_limits_and_sessions.py +++ b/packages/structured-data-models-client/tests/test_limits_and_sessions.py @@ -83,18 +83,20 @@ def test_an_explanation_covering_more_than_one_entity_is_refused(): assert 'one at a time' in str(error.value) -def test_an_explanation_over_the_smaller_context_budget_is_refused(): - """Explaining forces the smaller budget, which caps context at 1,000.""" +def test_explains_large_context(): + """An explanation over a 5,000-row context is sent. + + Assumptions: + - The payload scores one entity. + - The SDM NIM has no 1,000-row explanation limit. + """ request = _Gathered(payload(predict_rows=1, context_rows=5_000)) - with pytest.raises(UnsupportedRequestError) as error: - list( - RelationalAdapter().prediction_plan( - request, ('prediction', 'explanation') - ) + plans = list( + RelationalAdapter().prediction_plan( + request, ('prediction', 'explanation') ) - message = str(error.value) - assert 'num_context_samples' in message - assert '1,000' in message + ) + assert plans[0]['output']['fields'] == ['prediction', 'explanation'] def test_an_explanation_within_both_limits_is_allowed(): diff --git a/packages/structured-data-models-client/tests/test_request_and_dispatch.py b/packages/structured-data-models-client/tests/test_request_and_dispatch.py index be422884f..6274092a9 100644 --- a/packages/structured-data-models-client/tests/test_request_and_dispatch.py +++ b/packages/structured-data-models-client/tests/test_request_and_dispatch.py @@ -177,14 +177,20 @@ def test_a_task_kind_the_model_does_not_serve_is_refused(): check_request_against_model(request, KUMO_RELATIONAL, ('prediction',)) -def test_an_output_field_the_model_cannot_produce_is_refused_by_name(): +def test_unserved_output_named(): + """An output the model cannot produce is refused, naming it and the + outputs the model does serve. + + Assumptions: + - kumo-tabular does not serve rankings. + """ request = simple_request() with pytest.raises(UnsupportedRequestError) as error: check_request_against_model( - request, KUMO_TABULAR, ('prediction', 'explanation') + request, KUMO_TABULAR, ('prediction', 'rankings') ) message = str(error.value) - assert 'explanation' in message + assert 'rankings' in message assert 'probabilities' in message diff --git a/packages/structured-data-models-client/tests/test_tabular_explanations.py b/packages/structured-data-models-client/tests/test_tabular_explanations.py new file mode 100644 index 000000000..a4e969b47 --- /dev/null +++ b/packages/structured-data-models-client/tests/test_tabular_explanations.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Asking ``kumo-tabular`` for an explanation. + +The endpoint explains one scored row at a time, so the client refuses more +before sending, and hands the explanation it gets back to the caller. +""" + +from __future__ import annotations + +import pytest + +from sdm_client import UnsupportedRequestError +from sdm_client.adapters.tabular import TabularAdapter +from sdm_client.base import check_request_against_model +from sdm_client.models import KUMO_TABULAR + +EXPLAIN = ('prediction', 'explanation') + + +def response(**record) -> dict: + """A one-record ``kumo-tabular`` response carrying ``record``'s fields.""" + return { + 'id': 'r', + 'model': 'kumo-tabular', + 'predictions': [{'row_index': 0, 'prediction': 1, **record}], + 'metadata': {}, + } + + +@pytest.mark.parametrize('task', ['classification', 'regression']) +def test_allowed(tabular_request, task): + """kumo-tabular accepts explanation as an output on both of its tasks. + + Assumptions: + - The request scores one row. + """ + check_request_against_model( + tabular_request(task=task), KUMO_TABULAR, EXPLAIN + ) + + +def test_one_row_sent(tabular_request): + """An explanation over one row is sent with explanation in its fields. + + Assumptions: + - The request scores exactly one row. + """ + (envelope,) = TabularAdapter().prediction_plan(tabular_request(), EXPLAIN) + assert envelope['output']['fields'] == list(EXPLAIN) + + +def test_rows_refused(tabular_request): + """An explanation over two rows is refused before sending, naming the count. + + Assumptions: + - The endpoint explains one row at a time. + """ + with pytest.raises(UnsupportedRequestError) as error: + list(TabularAdapter().prediction_plan(tabular_request(rows=2), EXPLAIN)) + assert 'one at a time' in str(error.value) + assert error.value.details['scored'] == 2 + + +def test_rows_without_explanation(tabular_request): + """Several rows are sent when no explanation is asked for. + + Assumptions: + - The one-row limit applies only to explanations. + """ + (envelope,) = TabularAdapter().prediction_plan( + tabular_request(rows=2), ('prediction',) + ) + assert envelope['output']['fields'] == ['prediction'] + + +def test_pin_not_judged(tabular_request): + """Pinning a session context is not refused by the row limit. + + Assumptions: + - Pinning sends no predict rows, so the limit has nothing to count. + """ + pinned = TabularAdapter().session_plan(tabular_request(rows=2), EXPLAIN) + assert 'predict' not in pinned + + +def test_session_rows_refused(tabular_request): + """Session scoring refuses an explanation over two rows. + + Assumptions: + - Scoring sends the predict rows, so the limit applies. + """ + with pytest.raises(UnsupportedRequestError): + list( + TabularAdapter().session_prediction_plan( + tabular_request(rows=2), EXPLAIN + ) + ) + + +def test_result_carries_explanation(): + """The explanation in the response reaches the result unchanged. + + Assumptions: + - The endpoint returns one record with an explanation dictionary. + """ + explanation = {'format': 'sdm_gradient_v1', 'details': {'outputs': []}} + result = TabularAdapter().normalize( + response(explanation=explanation), EXPLAIN + ) + assert result.explanation == explanation + + +def test_result_without_explanation(): + """A response without an explanation leaves the result's explanation None. + + Assumptions: + - The record carries no explanation field. + """ + result = TabularAdapter().normalize(response(), ('prediction',)) + assert result.explanation is None diff --git a/qa/probe_tabular.py b/qa/probe_tabular.py index 56d0ae9dc..a903cadb4 100644 --- a/qa/probe_tabular.py +++ b/qa/probe_tabular.py @@ -331,11 +331,9 @@ def run(client): task='classification', ), model=KumoTabular(), - outputs=['explanation'], - ) - fail( - e, 'explanation was accepted by a model that does not serve it' + outputs=['rankings'], ) + fail(e, 'rankings was accepted by a model that does not serve it') except StructuredDataError as exc: note(e, str(exc)[:120]) diff --git a/skills/relational/SKILL.md b/skills/relational/SKILL.md index 72a62606b..20253fb3b 100644 --- a/skills/relational/SKILL.md +++ b/skills/relational/SKILL.md @@ -181,8 +181,7 @@ many calls. **The table limit is on the graph a request carries, not on the schema.** A schema may describe far more than fifteen tables; what counts is how many the -query actually reaches. An explanation has a second limit besides its one row: -it must fit the 1,000-row context budget. +query actually reaches. ## Asking for more than one answer each diff --git a/skills/tabular/SKILL.md b/skills/tabular/SKILL.md index 1038c758f..7d9434634 100644 --- a/skills/tabular/SKILL.md +++ b/skills/tabular/SKILL.md @@ -29,16 +29,17 @@ with SDMClient(url='http://localhost:8000') as client: | | `kumo-tabular` | |---|---| | Tasks | `classification`, `regression` | -| Outputs | `prediction`, `probabilities`, `quantiles` | +| Outputs | `prediction`, `probabilities`, `quantiles`, `explanation` | | Several tables | no | | Relationships | no | -| Explanations | no | +| Explanations | yes | | Sessions | yes | | Context rows | 10,000 | Anything outside that list belongs to the relational model. Asking for -`rankings`, an `explanation`, or a forecast here is refused before a request -is sent. +`rankings` or a forecast here is refused before a request is sent. An +`explanation` covers one scored row at a time, and asking for one over more +rows is refused before sending. Outputs narrow again by task, and that narrowing happens at the endpoint rather than in the client: `probabilities` on a classification, `quantiles` on a @@ -205,7 +206,8 @@ names what to change. | Refusal | Cause | |---|---| | a task the model does not serve | `forecasting`, `temporal_link_prediction` and the rest are relational | -| an output it does not produce | `rankings`, `explanation`, `embeddings`, `scores` | +| an output it does not produce | `rankings`, `embeddings`, `scores` | +| an explanation over more than one row | the endpoint explains one row at a time | | a context above the cap | more than 10,000 labelled rows in a direct `Request` | | relationships | this model reads one table; use `kumo-relational` | From 78b0a5ebb8f295258b34259ea05c1cd460df7348 Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Wed, 28 Oct 2026 17:47:02 -0700 Subject: [PATCH 02/25] Say kumo-tabular declares explanation where the docs still deny it The architecture table of what each model declares still marked `explanation` as absent for kumo-tabular, and a session test still said the endpoint only explains under the 'fast' run mode. Signed-off-by: Jana Gagacheva Co-Authored-By: Claude Opus 5.5 --- docs/about/architecture.md | 2 +- .../tests/test_limits_and_sessions.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/about/architecture.md b/docs/about/architecture.md index cda67c5dd..0462a720a 100644 --- a/docs/about/architecture.md +++ b/docs/about/architecture.md @@ -537,7 +537,7 @@ It is not what any one deployment answers; see below. | `scores` | ✅ | ❌ | | `rankings` | ✅ | ❌ | | `embeddings` | ✅ | ❌ | -| `explanation` | ✅ | ❌ | +| `explanation` | ✅ | ✅ | Asking a model for a field it does not declare fails client-side, naming what it does declare. diff --git a/packages/structured-data-models-client/tests/test_limits_and_sessions.py b/packages/structured-data-models-client/tests/test_limits_and_sessions.py index b1459e17d..ec85521d5 100644 --- a/packages/structured-data-models-client/tests/test_limits_and_sessions.py +++ b/packages/structured-data-models-client/tests/test_limits_and_sessions.py @@ -510,7 +510,7 @@ def test_a_session_refuses_a_request_asked_as_several_questions(): def test_a_session_scoring_call_carries_the_inference_block(): """The contract's SessionPredictionRequest lists `inference`; dropping it - made explanations unreachable through a session, because the endpoint only + made explanations unreachable through a session on a deployment that only explains when inference.run_mode is 'fast'.""" pytest.importorskip('sdm_relational_engine') from sdm_client.adapters.relational import _SESSION_SCORING From 793a2a8c410cf612eb3855516691b3bced9f3b66 Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Thu, 29 Oct 2026 13:01:44 -0700 Subject: [PATCH 03/25] Document per-row explanations on result.explanations Every scored row is explained, on both models, and result.explanations maps each row's key (id, (id, forecast_step), or row_index) to its entry. The docs, skills and changelog drop the one-row limit and result.explanation, describe the entry shape, related rows with keys and values, preprocessed columns and to_pandas() with hop, and name the server settings a NIM needs to explain (SDM_NUM_ESTIMATORS=1, and CUDA graph replay off on kumo-tabular), which otherwise answer 422. probe_tabular.py probes a multi-row explanation. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jana Gagacheva --- CHANGELOG.md | 21 ++- docs/about/architecture.md | 38 ++++- docs/reference/limits.md | 13 +- docs/reference/prediction-output.md | 147 +++++++++++++++++- qa/README.md | 2 +- qa/probe_tabular.py | 50 ++++++ skills/relational/SKILL.md | 24 ++- .../references/beyond-one-answer.md | 28 +++- skills/tabular/SKILL.md | 34 +++- 9 files changed, 337 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0606536fe..b4b7fd6b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,9 +29,15 @@ these packages. All five distributions share one version and one entry. installs on a laptop rather than on Linux only. The engine ships no source distribution, so those wheels remain the full set of platforms it supports: `structured-data-models-client` itself is pure Python and stays broader. -- `kumo-tabular` accepts `explanation` in `outputs`, over one scored row at a - time, and returns it on `result.explanation`. Asking for one over more rows - is refused before sending. +- `kumo-tabular` accepts `explanation` in `outputs`. +- `result.explanations` maps each explained row's key to its explanation: the + row's `id`, `(id, forecast_step)` on a forecast, or `row_index` when the + request has no entity id. Each entry is a plain dict of `outputs`, the + entity's own row under `instance`, and, on `kumo-relational`, each related + table under `related`, whose rows carry their primary key when the table + has one and always their values as sent. It is kept out of + `result.to_pandas()`; `result.explanations.to_pandas()` flattens it into one + line per row, output and column, with each row's `hop` from the entity. ### Changed @@ -48,6 +54,15 @@ these packages. All five distributions share one version and one entry. - A `kumo-relational` explanation is no longer refused above the 1,000-row context budget. That limit belonged to an earlier deployment; the Kumo Relational SDM NIM explains under either budget. +- An explanation covers every scored row, on both models. A `kumo-tabular` + explanation over more than one row and a `kumo-relational` one over more + than one entity are no longer refused before sending. + +### Removed + +- `result.explanation`, which held the first record's explanation only, and + the `{'by_forecast_step': ...}` it held for a forecast. Read + `result.explanations` instead. ### Fixed diff --git a/docs/about/architecture.md b/docs/about/architecture.md index 0462a720a..9c9be1fb1 100644 --- a/docs/about/architecture.md +++ b/docs/about/architecture.md @@ -500,7 +500,7 @@ result = client.predict( ) result.to_pandas() # always a DataFrame -result.explanation # None unless you asked for it +result.explanations # empty unless you asked for them ``` Explanations are an **output field**, not a method. Ask for one with @@ -564,7 +564,7 @@ So `scores`, `rankings`, `embeddings` and `explanation` are declared by the client and refused by that deployment on every task. That is a fact about this build, not about the fields: `explanation` has been measured working against another relational deployment, which returns an explanation the client parses -and hands back on `PredictionResult.explanation`. A deployment that refuses it is +and hands back on `PredictionResult.explanations`. A deployment that refuses it is narrower than the model family, not the client declaring something nothing serves. @@ -581,6 +581,40 @@ served either — a forecast is sent as a regression per step, and a bare so a deployment that gains or loses an output fails the suite rather than leaving the document wrong. +### Explanations are keyed by row, off the frame + +Every scored row is explained, and `result.explanations` maps each row's key +to its explanation: the `id` it carries in `to_pandas()`, `(id, +forecast_step)` for a forecast, and `row_index` when the request has no entity +key. The same rule holds on both models. The shape of an entry is in +[Prediction output](../reference/prediction-output.md#explanations). + +**Keys, not positions.** The endpoint locates every explained row by its +position in the tables as submitted. The caller never sees those tables: the +related ones are gathered by this client, and a keyless one is identified on +the wire only by the engine's own synthetic node id. So the client turns each +position back into what the caller can recognise. The predicted row's position +is dropped, since the mapping's key already names it. A related row carries its +primary key when its table has one, and always its values as sent, so a keyless +table's rows are still identified, by their contents. + +**Translated per call, before merging.** A request split across several calls +numbers its positions from zero in each call. Read after merging, the second +call's rows would point into the first call's tables. Each call's explanations +are translated while that call's own tables are still at hand, and only then +merged. + +**Kept off the frame.** An entry is nested: outputs by columns for the row +itself, and outputs by rows by columns for each related table. `to_pandas()` +stays one row per answer, as the task decides, and the explanations sit beside +it on the result. `result.explanations.to_pandas()` flattens them into a table +of their own when one is wanted, with each related row's `hop` from the entity +worked out from the relationships and foreign-key values the client sent. + +A forecast is explained a step at a time, under `(id, forecast_step)`, rather +than aggregated across steps: each step is its own row in `to_pandas()`, and +its explanation is keyed the same way. + --- ## 6. Adding a model diff --git a/docs/reference/limits.md b/docs/reference/limits.md index c6eae68ba..eb098458c 100644 --- a/docs/reference/limits.md +++ b/docs/reference/limits.md @@ -27,8 +27,6 @@ Both endpoints enforce sizes, and most of these are the relational one's. | calls a question asked in parts makes | relational | 200, one per candidate or step | at `materialize()`, as `sdm_client.decompose.MOST_PARTS` | | request body | relational | 30 MiB per call | at `materialize()` | | request body | relational, through Databricks model serving | 16 MB per call, the gateway's own decimal limit | before sending, on the wrapped envelope | -| an explanation | relational | exactly one entity | before sending | -| an explanation | tabular | exactly one row | before sending | | hops | relational | 6 | at `materialize()` | | tables | relational | 15 | against the graph the request carries, not the schema it was built from | @@ -38,6 +36,17 @@ deployment sets, such as the longest string a cell may hold or the most distinct values a column may have. Those protect a shared server, and are not applied when the model runs in your own process. +**An explanation has no row limit of its own.** Every scored row is +explained, on either model. It does depend on two server settings, which a +deployment chooses when its container starts and this client cannot see: + +- one estimator, `SDM_NUM_ESTIMATORS=1`, the default; +- on `kumo-tabular`, CUDA graph replay off: `SDM_TABULAR_CUDA_GRAPHS` unset, + not `1`. + +A deployment set otherwise answers a request for `explanation` with +`422 UNSUPPORTED_OUTPUT`. Ordinary predictions are unaffected. + A relational request for more entities than one call scores is not refused. `materialize()` gathers it in batches of at most that many, each with a context of its own, sampled around that batch's rows rather than copied from diff --git a/docs/reference/prediction-output.md b/docs/reference/prediction-output.md index 4179686f9..e03a4a068 100644 --- a/docs/reference/prediction-output.md +++ b/docs/reference/prediction-output.md @@ -6,7 +6,7 @@ explanation. ```python result = client.predict(request, model=KumoRelational()) result.to_pandas() # always a frame -result.explanation # None unless you asked for it +result.explanations # empty unless you asked for them result.metadata # whatever the endpoint reported ``` @@ -117,3 +117,148 @@ joined = to_score.reset_index(names='row_index').merge(answers, on='row_index') | `quantiles` | `quantile_`, one per level | | `scores`, `embeddings` | one column holding a list | | `rankings` | one column holding a list of dicts | + +## Explanations + +Ask for one by adding `'explanation'` to `outputs`. There is no row limit: +every scored row is explained. + +```python +result = client.predict(request, model=KumoRelational(), + outputs=['prediction', 'probabilities', 'explanation']) +``` + +The deployment must run with one estimator, and `kumo-tabular` with CUDA graph +replay off, or it answers `422 UNSUPPORTED_OUTPUT`; see +[Limits](limits.md). + +### Keys + +`result.explanations` maps a row's key to that row's explanation. It is kept +out of `result.to_pandas()`. There is no `result.explanation` and no +`{'by_forecast_step': ...}`. + +The key is whatever identifies the row in `to_pandas()`, by the same rule on +both models: + +| Request | Key | Example | +|---|---|---| +| has an entity id | the row's `id`, the same value as in `to_pandas()['id']` | `result.explanations['user_5']` | +| forecast | `(id, forecast_step)` | `result.explanations[('user_5', 3)]` | +| no entity id (single-table tabular) | `row_index` | `result.explanations[0]` | + +### One entry + +Each entry is a plain nested dict, the same shape for both models: + +```python +{ + 'outputs': [...], # the explained scores, in order + 'instance': { + 'columns': [...], # model's preprocessed numerical columns + 'gradients': [[...], ...], # [output][column] + }, + 'related': { # absent for kumo-tabular + '': { + 'keys': [...], # the table's primary key per row; absent if the table has none + 'rows': [{...}, ...], # each row's values as sent to the model + 'columns': [...], # model's preprocessed numerical columns + 'gradients': [[[...], ...], ...], # [output][row][column], rows aligned with keys and rows + }, + }, +} +``` + +`outputs` is copied from the endpoint: + +- classification gives every class probability in declared order, such as + `{'kind': 'probability', 'class': 'yes'}`; +- regression gives the returned prediction, + `{'kind': 'prediction', 'statistic': 'mean'}` (or `'median'`), plus each + returned quantile when `quantiles` is requested. + +A gradient is the change in that output score per unit change in that +preprocessed column. It is a diagnostic, not a causal effect or a +feature-importance ranking. + +`columns` are the model's preprocessed numerical columns, not the columns you +sent. Categoricals and datetimes may appear as derived numeric columns, and a +table without numerical features has an empty `columns`. + +### Which tables get gradients + +| Model | Tables with gradients | +|---|---| +| kumo-tabular | the one table: each predicted row's own columns | +| kumo-relational | the predicted entity's own table, plus every related table in the subgraph the client gathered for that entity (all tables reached within the hop limit, up to 6) | + +A `kumo-tabular` entry has no `related` key. + +Only the rows being predicted and their related rows get gradients. The +context rows, the labelled examples the model compares against, get none. + +Gradients are per row, not per path. A row reached through several paths, for +example a product bought in two of the user's orders, appears once. Its +gradient is its total effect through all of them, and the model gives no split +by path. + +### Related rows + +A related table's rows carry `keys`, the table's primary key per row, when the +table has one, and always `rows`, each row's values as sent to the model. A +keyless table has no `keys`, so its rows are identified by their values alone. +Keyed or keyless changes only how a row is labelled; both get gradients in the +same way. + +### Examples + +Values are illustrative. `kumo-tabular`, request with an id column: + +```python +result.explanations['cust_17'] +# {'outputs': [{'kind': 'probability', 'class': 'no'}, +# {'kind': 'probability', 'class': 'yes'}], +# 'instance': {'columns': ['age', 'balance'], +# 'gradients': [[0.01, -0.03], [-0.01, 0.03]]}} +``` + +`kumo-relational`, user 5 with one order and two login events. `orders` has a +primary key (`order_id`), so its rows carry `keys`. `events` has none, so its +rows are identified by their values alone. Both are part of the same entry: + +```python +result.explanations['user_5'] +# {'outputs': [{'kind': 'probability', 'class': 'no'}, +# {'kind': 'probability', 'class': 'yes'}], +# 'instance': {'columns': ['tenure_days'], +# 'gradients': [[-0.05], [0.05]]}, +# 'related': { +# 'orders': {'keys': ['order_9003'], +# 'rows': [{'user_id': 'user_5', 'order_id': 'order_9003', 'amount': 12.0}], +# 'columns': ['amount'], +# 'gradients': [[[-0.02]], [[0.02]]]}, +# 'events': {'rows': [{'user_id': 'user_5', 'kind': 'login', 'at': '2026-10-01T09:00:00'}, +# {'user_id': 'user_5', 'kind': 'login', 'at': '2026-10-03T18:20:00'}], +# 'columns': ['kind_login', 'at_dayofweek'], # model-derived names, illustrative +# 'gradients': [[[0.01, 0.00], [0.02, -0.01]], +# [[-0.01, 0.00], [-0.02, 0.01]]]}}} +``` + +### As one table + +`result.explanations.to_pandas()` gives one line per (row, output, column), +across all entities: + +``` +id table hop key row output column gradient +user_5 users 0 user_5 0 P(yes) tenure_days 0.05 +user_5 orders 1 order_9003 0 P(yes) amount 0.02 +user_5 products 2 prod_77 0 P(yes) price 0.004 +user_5 events 1 - 1 P(yes) kind_login 0.02 +``` + +- `hop` is how many links the row is from the entity, 0 for the entity + itself. The client works it out from the relationships and foreign-key + values it sent. +- `key` is the row's primary key, or empty for a keyless table. `row` is the + row's position in that entity's `related[table]['rows']`. diff --git a/qa/README.md b/qa/README.md index 1530af651..bed9cfe0a 100644 --- a/qa/README.md +++ b/qa/README.md @@ -71,7 +71,7 @@ anything that comes back red is something that used to be broken. | Suite | Probes | |---|---| -| `probe_tabular.py` | 24 | +| `probe_tabular.py` | 25 | | `probe_relational.py` | 27 | | `probe_connectors.py` | 29 | | `probe_edges.py` | 27 | diff --git a/qa/probe_tabular.py b/qa/probe_tabular.py index a903cadb4..8cc073a65 100644 --- a/qa/probe_tabular.py +++ b/qa/probe_tabular.py @@ -337,6 +337,56 @@ def run(client): except StructuredDataError as exc: note(e, str(exc)[:120]) + with probe( + AREA, + 'an explanation over several rows', + 'one entry per scored row, keyed by row_index, no related tables', + ) as e: + ctx, sc = split(data.adult(), 200, 4) + result = client.predict( + Request( + context=ctx, + predict=sc.drop(columns=['income']), + target='income', + task='classification', + ), + model=KumoTabular(), + outputs=['prediction', 'probabilities', 'explanation'], + ) + frame = result.to_pandas() + explained = result.explanations + if set(explained) != set(frame['row_index']): + fail( + e, + f'explained {sorted(explained)}, ' + f'scored {sorted(frame["row_index"])}', + ) + elif any('related' in entry for entry in explained.values()): + fail(e, 'a kumo-tabular entry carries related tables') + elif any( + len(entry['instance']['gradients']) != len(entry['outputs']) + or any( + len(per_output) != len(entry['instance']['columns']) + for per_output in entry['instance']['gradients'] + ) + for entry in explained.values() + ): + fail(e, 'gradients are not [output][column]') + else: + flat = explained.to_pandas() + if 'explanation' in frame.columns: + fail(e, 'the explanation leaked into to_pandas()') + elif 'hop' not in flat.columns or set(flat['hop']) != {0}: + fail(e, f'to_pandas() hops: {list(flat.get("hop", []))[:8]}') + else: + first = explained[min(explained)] + note( + e, + f'{len(explained)} entries, ' + f'{len(first["instance"]["columns"])} columns each, ' + f'{len(flat)} lines flat', + ) + # --- correlation ------------------------------------------------------ with probe( AREA, 'row_index joins one-to-one', 'every scored row is answered once' diff --git a/skills/relational/SKILL.md b/skills/relational/SKILL.md index 20253fb3b..7ad3b0c36 100644 --- a/skills/relational/SKILL.md +++ b/skills/relational/SKILL.md @@ -159,7 +159,6 @@ would break rather than letting the endpoint return a 500. | context rows, with `num_context_samples` | `10,000` | | rows scored at once | `10,000` | | rows scored for a link prediction | `200` | -| rows scored with an explanation | `1` | | tables in the gathered graph | `15` | | hops gathered | `6` | | classes | `10` | @@ -186,10 +185,25 @@ query actually reaches. ## Asking for more than one answer each A ranking is a call per candidate and a forecast a call per step; an -explanation is an output, over one scored row. Each has limits and refusals of -its own, and they are in +explanation is an output, over every scored row. Each has limits and refusals +of its own, and they are in [`references/beyond-one-answer.md`](references/beyond-one-answer.md). +An explanation is kept out of `to_pandas()`, on `result.explanations`, a +mapping from each row's key to its explanation: the entity's `id`, or +`(id, forecast_step)` on a forecast. One entry covers the entity's own row and +every related row gathered for it: + +```python +entry = result.explanations['user_5'] # the entity whose id is 'user_5' +entry['instance']['gradients'] # [output][column], the entity's own row +orders = entry['related']['orders'] # one block per related table +orders['keys'] # each row's primary key; absent if the table has none +orders['rows'] # each row's values as sent +orders['gradients'] # [output][row][column] +result.explanations.to_pandas() # one line per (row, output, column), with its hop +``` + ## Asking the same context more than once `fit()` uploads a context once and pins it in the endpoint's memory; every @@ -272,6 +286,6 @@ change. The ones worth knowing: | more than 100,000 entities at once | every batch is built before any is sent; ask about fewer at a time | | more than 100 calls at once | a context is gathered per call; ask about fewer entities, or raise `rows_per_call` if you lowered it | | more entities than one call scores, on a ranking or a forecast | each candidate or step is already a call, and batching one would send that many times as many; ask about fewer at a time | -| an output this question cannot give | `rankings` on a regression, `explanation` on a ranking or on more than one row | -| `422 UNSUPPORTED_OUTPUT` | the client sent an output the deployment does not serve, such as `explanation` | +| an output this question cannot give | `rankings` on a regression, `explanation` on a ranking | +| `422 UNSUPPORTED_OUTPUT` | the client sent an output the deployment does not serve, such as `explanation`, or asked for `explanation` of a deployment not started with one estimator, `SDM_NUM_ESTIMATORS=1` | | an aggregation the model does not answer | `COUNT_DISTINCT`, `FIRST`, `LAST`, or any aggregation of a category | diff --git a/skills/relational/references/beyond-one-answer.md b/skills/relational/references/beyond-one-answer.md index 12fe77df9..d20ef6c00 100644 --- a/skills/relational/references/beyond-one-answer.md +++ b/skills/relational/references/beyond-one-answer.md @@ -113,6 +113,28 @@ labels come from, so a ranking and a forecast are asked with `predict()` and `fit()` refuses them. An explanation is asked for as an output, `outputs=[..., 'explanation']`, never -an `explain=` argument, and read from `result.explanation`. It covers one -scored row at a time. A ranking refuses it locally, and the shipping endpoint -refuses it everywhere else, so treat `result.explanation` as possibly `None`. +an `explain=` argument, and read from `result.explanations`. It covers every +scored row, with no row limit. `result.explanations` maps each row's key to its +explanation, the same key that identifies the row in `to_pandas()`: the +entity's `id`, or `(id, forecast_step)` on a forecast, since a forecast is +keyed by the pair. It is kept out of `to_pandas()` itself. + +An entry holds `outputs`, the explained scores in order; `instance`, the +entity's own row, with `columns` and `gradients` as `[output][column]`; and +`related`, one block per related table the client gathered for that entity. +A block carries `keys`, each row's primary key, when the table has one, and +always `rows`, each row's values as sent, with `columns` and `gradients` as +`[output][row][column]`, aligned with those rows. A keyless table's rows are +identified by their values alone, and get gradients the same way. + +`columns` are the model's preprocessed numerical columns: categoricals and +datetimes may appear as derived columns, and a table without numerical +features gives an empty list. The context rows get no gradients. A row reached +through several paths appears once, with its total effect through all of them. +`result.explanations.to_pandas()` flattens every entry into one line per (row, +output, column), with `hop`, how many links the row is from the entity. + +A ranking refuses an explanation locally, and the shipping endpoint refuses it +everywhere else, so treat `result.explanations` as possibly empty. A deployment +that serves explanations has to run one estimator, `SDM_NUM_ESTIMATORS=1`, and +answers `422 UNSUPPORTED_OUTPUT` otherwise. diff --git a/skills/tabular/SKILL.md b/skills/tabular/SKILL.md index 7d9434634..a2134e3de 100644 --- a/skills/tabular/SKILL.md +++ b/skills/tabular/SKILL.md @@ -38,8 +38,7 @@ with SDMClient(url='http://localhost:8000') as client: Anything outside that list belongs to the relational model. Asking for `rankings` or a forecast here is refused before a request is sent. An -`explanation` covers one scored row at a time, and asking for one over more -rows is refused before sending. +`explanation` covers every scored row: see [Explanations](#explanations). Outputs narrow again by task, and that narrowing happens at the endpoint rather than in the client: `probabilities` on a classification, `quantiles` on a @@ -198,6 +197,35 @@ A session pins the context into one process's memory on the endpoint, so a deployment behind several replicas needs requests for one session to reach the same one. +## Explanations + +Add `'explanation'` to `outputs` and every scored row is explained. There is no +row limit. The explanations are kept out of `to_pandas()`, on +`result.explanations`, a mapping from each row's key to its explanation: the +row's `id` when the request has one, otherwise its `row_index`. + +```python +entry = result.explanations[0] # the row scored at row_index 0 +entry['outputs'] # the explained scores, in order +entry['instance']['columns'] # the model's preprocessed numerical columns +entry['instance']['gradients'] # [output][column] +result.explanations.to_pandas() # one line per (row, output, column) +``` + +An entry here has no `related` key: this model reads one table. `columns` are +the columns as the model sees them, not as you sent them. Categoricals and +datetimes may appear as derived columns, and a table without numerical +features gives an empty list. A gradient is the change in that output score +per unit change in that column: a diagnostic, not a causal effect or a +feature-importance ranking. The whole shape is under Explanations in +[`docs/reference/prediction-output.md`](../../docs/reference/prediction-output.md). + +The deployment has to be started with one estimator, `SDM_NUM_ESTIMATORS=1`, +and CUDA graph replay off, `SDM_TABULAR_CUDA_GRAPHS` unset. Both are server +settings this client cannot see. A deployment started otherwise answers +`422 UNSUPPORTED_OUTPUT` to a request for `explanation`, and ordinary +predictions are unaffected. + ## When it refuses Every refusal is a `StructuredDataError` or one of its subclasses, and each @@ -207,7 +235,7 @@ names what to change. |---|---| | a task the model does not serve | `forecasting`, `temporal_link_prediction` and the rest are relational | | an output it does not produce | `rankings`, `embeddings`, `scores` | -| an explanation over more than one row | the endpoint explains one row at a time | +| `422 UNSUPPORTED_OUTPUT` on `explanation` | the deployment runs more than one estimator or CUDA graph replay; see [Explanations](#explanations) | | a context above the cap | more than 10,000 labelled rows in a direct `Request` | | relationships | this model reads one table; use `kumo-relational` | From 0da19289ed710000be6b16d5605f40f11859935a Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Thu, 29 Oct 2026 13:10:18 -0700 Subject: [PATCH 04/25] Drop the one-row limit on explanations The NIM now explains every predict row, one explanation per record, so the client no longer refuses an explanation over more than one row: explanation_predict_rows leaves TABULAR_LIMITS and RELATIONAL_LIMITS, and the tabular row check and the relational one-entity check go with it. Tests that asserted those refusals are removed or now assert the rows are sent, and the skills limits table no longer maps a row to the removed limit. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jana Gagacheva --- .../src/sdm_client/adapters/relational.py | 24 +--------- .../src/sdm_client/adapters/tabular.py | 26 +--------- .../src/sdm_client/models.py | 2 - .../tests/test_limits_and_absent_outputs.py | 16 ------- .../tests/test_limits_and_sessions.py | 20 -------- .../tests/test_tabular_explanations.py | 47 ++++++------------- tests/test_skills.py | 1 - 7 files changed, 16 insertions(+), 120 deletions(-) diff --git a/packages/structured-data-models-client/src/sdm_client/adapters/relational.py b/packages/structured-data-models-client/src/sdm_client/adapters/relational.py index 3691cbcc4..a4e84bd8a 100644 --- a/packages/structured-data-models-client/src/sdm_client/adapters/relational.py +++ b/packages/structured-data-models-client/src/sdm_client/adapters/relational.py @@ -126,7 +126,7 @@ def prediction_plan( sends: frozenset[str] = _SENDS_EVERYTHING, ) -> Iterator[dict[str, Any]]: payloads = self._payloads(request) - self._preflight(payloads, outputs, request.query, sends) + self._preflight(payloads, request.query, sends) for payload in payloads: envelope = dict(payload) envelope['model'] = self.spec.model_id @@ -358,15 +358,11 @@ def close(self) -> None: def _preflight( self, payloads: tuple[dict[str, Any], ...], - outputs: tuple[str, ...], asked: str | None = None, sends: frozenset[str] = _SENDS_EVERYTHING, ) -> None: """Refuse what the endpoint enforces, before spending the round trip. - An explanation is the narrow case: the model explains one entity at a - time, and the endpoint refuses more, so that limit is checked here. - `sends` names the halves this call puts on the wire. A session pins its context at `fit()` and a scoring call carries only rows, so judging the context there refuses a request over rows that are thrown away, while @@ -378,24 +374,6 @@ def _preflight( self._check_context_is_not_empty(payloads) self._check_class_count(payloads, limits) self._check_target_varies(payloads, asked) - if 'explanation' not in outputs: - return - - scored = sum( - len( - payload.get('predict', {}) - .get('instance_table', {}) - .get('rows', []) - ) - for payload in payloads - ) - allowed = limits['explanation_predict_rows'] - if 'predict' in sends and scored != allowed: - raise UnsupportedRequestError( - f'an explanation covers exactly {allowed} entity, and this ' - f'request scores {scored}; ask for one at a time', - details={'model': self.spec.model_id, 'scored': scored}, - ) def _check_class_count( self, payloads: tuple[dict[str, Any], ...], limits: Mapping[str, int] diff --git a/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py b/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py index 5b4ce5b4d..bb654d14e 100644 --- a/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py +++ b/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py @@ -92,9 +92,7 @@ def prediction_plan( have to carry the same context, so splitting would multiply the upload without raising the cap. """ - envelope = self._envelope(request, outputs) - self._check_explanation_rows(request, outputs) - yield envelope + yield self._envelope(request, outputs) def session_plan( self, request: Request, outputs: tuple[str, ...] @@ -125,7 +123,6 @@ def session_prediction_plan( raise _a_row_the_pinned_schema_cannot_hold( request, declared, error ) from error - self._check_explanation_rows(request, outputs) yield { key: envelope[key] for key in _SESSION_SCORING if key in envelope } @@ -189,27 +186,6 @@ def close(self) -> None: r"""Nothing to release: this adapter sends through the client's own transport and opens nothing of its own.""" - def _check_explanation_rows( - self, request: Request, outputs: tuple[str, ...] - ) -> None: - r"""Refuse an explanation over more rows than the endpoint explains. - - The endpoint explains one row at a time and refuses more with - ``UNSUPPORTED_OUTPUT``, so the limit is named here before anything is - sent. Only the rows being scored count: a session's pinned context is - never judged by it. - """ - if 'explanation' not in outputs: - return - allowed = self.spec.runtime.limits['explanation_predict_rows'] - scored = len(request.predict.instance_table) - if scored != allowed: - raise UnsupportedRequestError( - f'an explanation covers exactly {allowed} row, and this ' - f'request scores {scored}; ask for one at a time', - details={'model': self.spec.model_id, 'scored': scored}, - ) - def _envelope( self, request: Request, diff --git a/packages/structured-data-models-client/src/sdm_client/models.py b/packages/structured-data-models-client/src/sdm_client/models.py index 3e0a88226..2e360f229 100644 --- a/packages/structured-data-models-client/src/sdm_client/models.py +++ b/packages/structured-data-models-client/src/sdm_client/models.py @@ -66,7 +66,6 @@ 'context_rows_max': 10_000, 'predict_rows_max': 10_000, 'predict_rows_temporal_link_prediction': 200, - 'explanation_predict_rows': 1, 'hops_max': 6, 'tables_max': 15, 'classes_max': 10, @@ -89,7 +88,6 @@ # large context to be told the limit at the far end. TABULAR_LIMITS = { 'context_rows_max': 10_000, - 'explanation_predict_rows': 1, } KUMO_TABULAR = ModelSpec( diff --git a/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py b/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py index f4ce97362..d2e1fcc40 100644 --- a/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py +++ b/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py @@ -558,19 +558,3 @@ def test_a_session_scoring_call_is_not_judged_on_a_context_it_discards(): sent = list(adapter.session_prediction_plan(request, ('prediction',))) assert sent and 'context' not in sent[0] - - -def test_pinning_a_context_is_not_judged_on_rows_it_does_not_send(): - """A session payload carries no predict rows, so the one-entity - explanation rule has nothing to count.""" - from sdm_client.adapters.relational import RelationalAdapter - - adapter = RelationalAdapter() - request = gathered(context_rows=50, run_mode='fast', predict_rows=3) - request.query = None - - with pytest.raises(StructuredDataError, match='exactly 1 entity'): - list(adapter.prediction_plan(request, ('prediction', 'explanation'))) - - pinned = adapter.session_plan(request, ('prediction', 'explanation')) - assert 'predict' not in pinned diff --git a/packages/structured-data-models-client/tests/test_limits_and_sessions.py b/packages/structured-data-models-client/tests/test_limits_and_sessions.py index ec85521d5..005eaa844 100644 --- a/packages/structured-data-models-client/tests/test_limits_and_sessions.py +++ b/packages/structured-data-models-client/tests/test_limits_and_sessions.py @@ -57,7 +57,6 @@ def test_the_declared_limits_are_pinned_to_what_was_read_off_the_endpoint(): assert RELATIONAL_LIMITS['context_rows_max'] == 10_000 assert RELATIONAL_LIMITS['predict_rows_max'] == 10_000 assert RELATIONAL_LIMITS['predict_rows_temporal_link_prediction'] == 200 - assert RELATIONAL_LIMITS['explanation_predict_rows'] == 1 assert RELATIONAL_LIMITS['hops_max'] == 6 assert RELATIONAL_LIMITS['tables_max'] == 15 @@ -71,18 +70,6 @@ def test_num_hops_must_be_a_whole_number(bad): assert error.value.code == 'INVALID_REQUEST' -def test_an_explanation_covering_more_than_one_entity_is_refused(): - """The model explains one entity at a time; more gets an opaque 500.""" - request = _Gathered(payload(predict_rows=4, context_rows=100)) - with pytest.raises(UnsupportedRequestError) as error: - list( - RelationalAdapter().prediction_plan( - request, ('prediction', 'explanation') - ) - ) - assert 'one at a time' in str(error.value) - - def test_explains_large_context(): """An explanation over a 5,000-row context is sent. @@ -246,13 +233,6 @@ def test_the_preflight_runs_through_a_real_materialized_request(): assert len(plans) == 1 assert plans[0]['model'] == 'kumo-relational' - with pytest.raises(UnsupportedRequestError): - list( - RelationalAdapter().prediction_plan( - request, ('prediction', 'explanation') - ) - ) - def gathered_in_calls(*, calls: int, context_rows: int): """A relational request already gathered as ``calls`` envelopes.""" diff --git a/packages/structured-data-models-client/tests/test_tabular_explanations.py b/packages/structured-data-models-client/tests/test_tabular_explanations.py index a4e969b47..743af6bbc 100644 --- a/packages/structured-data-models-client/tests/test_tabular_explanations.py +++ b/packages/structured-data-models-client/tests/test_tabular_explanations.py @@ -12,7 +12,6 @@ import pytest -from sdm_client import UnsupportedRequestError from sdm_client.adapters.tabular import TabularAdapter from sdm_client.base import check_request_against_model from sdm_client.models import KUMO_TABULAR @@ -52,52 +51,34 @@ def test_one_row_sent(tabular_request): assert envelope['output']['fields'] == list(EXPLAIN) -def test_rows_refused(tabular_request): - """An explanation over two rows is refused before sending, naming the count. +def test_rows_sent(tabular_request): + """An explanation over two rows is sent with explanation in its fields. Assumptions: - - The endpoint explains one row at a time. - """ - with pytest.raises(UnsupportedRequestError) as error: - list(TabularAdapter().prediction_plan(tabular_request(rows=2), EXPLAIN)) - assert 'one at a time' in str(error.value) - assert error.value.details['scored'] == 2 - - -def test_rows_without_explanation(tabular_request): - """Several rows are sent when no explanation is asked for. - - Assumptions: - - The one-row limit applies only to explanations. + - The endpoint explains every scored row. """ (envelope,) = TabularAdapter().prediction_plan( - tabular_request(rows=2), ('prediction',) + tabular_request(rows=2), EXPLAIN ) - assert envelope['output']['fields'] == ['prediction'] - + assert envelope['output']['fields'] == list(EXPLAIN) -def test_pin_not_judged(tabular_request): - """Pinning a session context is not refused by the row limit. - Assumptions: - - Pinning sends no predict rows, so the limit has nothing to count. - """ +def test_pin_sends_no_rows(tabular_request): + """Pinning a session context for explanations sends no predict rows.""" pinned = TabularAdapter().session_plan(tabular_request(rows=2), EXPLAIN) assert 'predict' not in pinned -def test_session_rows_refused(tabular_request): - """Session scoring refuses an explanation over two rows. +def test_session_rows_sent(tabular_request): + """Session scoring sends an explanation over two rows. Assumptions: - - Scoring sends the predict rows, so the limit applies. + - The endpoint explains every scored row. """ - with pytest.raises(UnsupportedRequestError): - list( - TabularAdapter().session_prediction_plan( - tabular_request(rows=2), EXPLAIN - ) - ) + (envelope,) = TabularAdapter().session_prediction_plan( + tabular_request(rows=2), EXPLAIN + ) + assert envelope['output']['fields'] == list(EXPLAIN) def test_result_carries_explanation(): diff --git a/tests/test_skills.py b/tests/test_skills.py index 08b349c96..b364e0296 100644 --- a/tests/test_skills.py +++ b/tests/test_skills.py @@ -2291,7 +2291,6 @@ def test_the_capability_table_is_the_clients_own(name, model_name): 'rows scored for a link prediction': ( 'predict_rows_temporal_link_prediction' ), - 'rows scored with an explanation': 'explanation_predict_rows', 'tables in the gathered graph': 'tables_max', 'hops gathered': 'hops_max', 'classes': 'classes_max', From 23db8a188bfae009bec572123e29b6e6ad6cf6e3 Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Thu, 29 Oct 2026 13:10:40 -0700 Subject: [PATCH 05/25] Give each explained row its own explanation on result.explanations result.explanation kept the first record's explanation and dropped the rest, and a forecast gathered its steps under {'by_forecast_step': ...}. Both are replaced by result.explanations, a mapping from the key that names a row in to_pandas() -- its id, (id, forecast_step) for a forecast, or row_index when the request has no entity key -- to that row's explanation, kept out of the frame and empty when none was asked for. Each entry is read beside the payload of the call that produced it, so a split request's positions, which restart at 0 in every call, are translated before the calls are merged. The instance block drops its row_indices and its gradients become [output][column]. A related row is given back as its primary key, when the table has one of its own, and always as its values as sent; kumo-tabular entries have no related key. Explanations.to_pandas() flattens every gradient to one line, with each related row's hop from the entity worked out from the relationships and key values sent. Explanations is exported, since result.explanations hands it out. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jana Gagacheva --- .../src/sdm_client/__init__.py | 3 +- .../src/sdm_client/adapters/relational.py | 186 +++++++++++++++- .../src/sdm_client/adapters/tabular.py | 10 +- .../src/sdm_client/client.py | 28 ++- .../src/sdm_client/result.py | 207 ++++++++++++++++-- .../src/sdm_client/wire/explanations.py | 91 ++++++++ .../tests/test_envelope_and_result.py | 11 +- .../tests/test_limits_and_absent_outputs.py | 8 +- .../tests/test_tabular_explanations.py | 33 ++- tests/golden/public_api.json | 16 +- 10 files changed, 538 insertions(+), 55 deletions(-) create mode 100644 packages/structured-data-models-client/src/sdm_client/wire/explanations.py diff --git a/packages/structured-data-models-client/src/sdm_client/__init__.py b/packages/structured-data-models-client/src/sdm_client/__init__.py index c9d4002a0..561991ba2 100644 --- a/packages/structured-data-models-client/src/sdm_client/__init__.py +++ b/packages/structured-data-models-client/src/sdm_client/__init__.py @@ -33,7 +33,7 @@ from sdm_client.models import KumoRelational, KumoTabular, Model from sdm_client.query import PredictiveQuery from sdm_client.request import Request, Split, Target, Task -from sdm_client.result import PredictionResult +from sdm_client.result import Explanations, PredictionResult from sdm_client.schema import ( Column, Relationship, @@ -45,6 +45,7 @@ __all__ = [ 'SDMWarning', 'Column', + 'Explanations', 'FittedModel', 'KumoRelational', 'KumoTabular', diff --git a/packages/structured-data-models-client/src/sdm_client/adapters/relational.py b/packages/structured-data-models-client/src/sdm_client/adapters/relational.py index a4e84bd8a..93bb939e5 100644 --- a/packages/structured-data-models-client/src/sdm_client/adapters/relational.py +++ b/packages/structured-data-models-client/src/sdm_client/adapters/relational.py @@ -49,6 +49,7 @@ from sdm_client.result import PredictionResult from sdm_client.wire import as_declared from sdm_client.wire.answers import as_declared_target +from sdm_client.wire.explanations import RelatedReader, explanations_of from sdm_client.wire.inference import typed_inference # What to ask for when the caller named nothing, by task kind. Asking for @@ -282,16 +283,18 @@ def answered( request_id: str = '', metadata: Mapping[str, Any] | None = None, ) -> PredictionResult: - explanation = next( - (item.explanation for item in items if item.explanation), None - ) + named = as_declared_target(_named_entities(items, payload), payload) return PredictionResult( model=model, request_id=request_id, - items=as_declared_target(_named_entities(items, payload), payload), + items=named, metadata=dict(metadata or {}), outputs=outputs, - explanation=explanation, + # Read against this call's own tables, whose positions restart at + # 0 in every call of a split request. + explanations=explanations_of( + named, _related_rows_of(payload or {}) + ), ) def explain_error( @@ -822,3 +825,176 @@ def _key_dtype(payload: dict[str, Any], reference: str) -> str | None: .get('dtype') ) return None + + +#: The engine's own transport columns on a related table: the instance a row +#: was sampled for, and the key it numbers a keyless table's rows by +#: (`rfm/inference/payload.py`). Neither is a value the caller sent, so a +#: related row is given back without them. +_INSTANCE_ID = 'instance_id' +_SYNTHETIC_KEY = '__node_id' + + +def _related_rows_of(payload: Mapping[str, Any]) -> RelatedReader: + r"""Read a record's related blocks as rows of this call, not positions. + + The endpoint names each related row by its position in the table as this + call sent it, which the caller never saw and which a split request + restarts at 0 in every call. Each is given back as its primary key, when + the table has one of the caller's, and always as its values as sent. + + How far each row is from the entity is worked out once per call, and only + if some record is explained. + """ + schema = payload.get('schema') or {} + sent = (payload.get('predict') or {}).get('related_tables') or {} + hops: dict[str, list[int | None]] | None = None + + def read( + blocks: Mapping[str, Any], + ) -> tuple[dict[str, Any], dict[str, list[int | None]]]: + nonlocal hops + if hops is None: + hops = _hops_from_the_entity(payload) + related: dict[str, Any] = {} + away: dict[str, list[int | None]] = {} + for name, block in blocks.items(): + table = sent.get(name) or {} + columns = list(table.get('columns') or []) + rows = [table['rows'][index] for index in block['row_indices']] + described = (schema.get('related_tables') or {}).get(name) or {} + key = _caller_key(described) + hidden = {_INSTANCE_ID, *_synthetic_keys(described)} + read_back: dict[str, Any] = {} + if key is not None and key in columns: + dtype = ( + (described.get('columns') or {}).get(key, {}).get('dtype') + ) + read_back['keys'] = [ + as_declared(row[columns.index(key)], dtype) for row in rows + ] + read_back['rows'] = [ + { + column: value + for column, value in zip(columns, row, strict=True) + if column not in hidden + } + for row in rows + ] + read_back['columns'] = list(block.get('columns') or []) + read_back['gradients'] = block.get('gradients') or [] + related[name] = read_back + away[name] = [hops[name][index] for index in block['row_indices']] + return related, away + + return read + + +def _key_columns(described: Mapping[str, Any]) -> list[str]: + r"""A related table's primary key, less the instance that scopes it.""" + key = described.get('primary_key') or [] + key = [key] if isinstance(key, str) else list(key) + return [column for column in key if column != _INSTANCE_ID] + + +def _synthetic_keys(described: Mapping[str, Any]) -> list[str]: + r"""The key columns the engine made up for a table that had none.""" + return [ + column + for column in _key_columns(described) + if column.startswith(_SYNTHETIC_KEY) + ] + + +def _caller_key(described: Mapping[str, Any]) -> str | None: + r"""The table's own primary key, or ``None`` if it has none of its own.""" + key = _key_columns(described) + if len(key) != 1 or _synthetic_keys(described): + return None + return key[0] + + +def _hops_from_the_entity( + payload: Mapping[str, Any], +) -> dict[str, list[int | None]]: + r"""How many links each related row of this call is from its entity. + + Read from the relationships and key values sent, the same ones the model + joins on. A relationship with no ``source_table`` starts at the instance + table, and the rows it reaches are the entity's own: 0. Every other one is + followed either way, and a row is as far as its nearest path. Joins stay + within one instance, as the engine scopes them, so two entities sharing a + product never meet through it. + + Positions are this call's, one list per related table of its predict half. + ``None`` for a row no relationship reaches from an entity. + """ + schema = payload.get('schema') or {} + predict = payload.get('predict') or {} + instance = predict.get('instance_table') or {} + tables = predict.get('related_tables') or {} + hops: dict[str, list[int | None]] = { + name: [None] * len(table.get('rows') or []) + for name, table in tables.items() + } + linked: dict[tuple[str, int], list[tuple[str, int]]] = {} + reached: list[tuple[str, int]] = [] + for link in schema.get('relationships') or []: + target = link.get('target_table') + if target not in tables: + continue + by_value: dict[Any, list[int]] = {} + for position, value in enumerate( + _joined_on(tables[target], link.get('target_columns') or []) + ): + if value is not None: + by_value.setdefault(value, []).append(position) + source = link.get('source_table') + columns = link.get('source_columns') or [] + if source is None: + for value in _joined_on(instance, columns): + reached += [(target, at) for at in by_value.get(value, ())] + continue + if source not in tables: + continue + for position, value in enumerate(_joined_on(tables[source], columns)): + for at in by_value.get(value, ()): + linked.setdefault((source, position), []).append((target, at)) + linked.setdefault((target, at), []).append((source, position)) + + for name, position in reached: + hops[name][position] = 0 + frontier = reached + distance = 0 + while frontier: + distance += 1 + following = [] + for row in frontier: + for name, position in linked.get(row, ()): + if hops[name][position] is None: + hops[name][position] = distance + following.append((name, position)) + frontier = following + return hops + + +def _joined_on(table: Mapping[str, Any], columns: Sequence[str]) -> list[Any]: + r"""What each row of ``table`` joins on through ``columns``. + + The instance a row belongs to, and its values in ``columns``; ``None`` for + a row holding a null there, which joins nothing. Empty when the table does + not carry every one of ``columns``. + """ + names = list(table.get('columns') or []) + if not columns or any(column not in names for column in columns): + return [] + at = [names.index(column) for column in columns] + scope = names.index(_INSTANCE_ID) if _INSTANCE_ID in names else None + joined: list[Any] = [] + for row in table.get('rows') or []: + values = tuple(row[index] for index in at) + if any(value is None for value in values): + joined.append(None) + continue + joined.append((None if scope is None else row[scope], values)) + return joined diff --git a/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py b/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py index bb654d14e..dd4d693a0 100644 --- a/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py +++ b/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py @@ -50,6 +50,7 @@ widen_tfm_dtype, ) from sdm_client.wire.answers import as_declared_target +from sdm_client.wire.explanations import explanations_of from sdm_client.wire.inference import typed_inference # Output fields to ask for when the caller named none, by task kind. Asking for @@ -163,16 +164,15 @@ def answered( request_id: str = '', metadata: Mapping[str, Any] | None = None, ) -> PredictionResult: - explanation = next( - (item.explanation for item in items if item.explanation), None - ) + named = as_declared_target(_named_entities(items, payload), payload) return PredictionResult( model=model, request_id=request_id, - items=as_declared_target(_named_entities(items, payload), payload), + items=named, metadata=dict(metadata or {}), outputs=outputs, - explanation=explanation, + # One table, so an entry has no related rows to name. + explanations=explanations_of(named), ) def explain_error( diff --git a/packages/structured-data-models-client/src/sdm_client/client.py b/packages/structured-data-models-client/src/sdm_client/client.py index 18c52bba7..ab5c20fec 100644 --- a/packages/structured-data-models-client/src/sdm_client/client.py +++ b/packages/structured-data-models-client/src/sdm_client/client.py @@ -48,7 +48,7 @@ _envelope_rows, warn_of_rows_scored_from_their_own_answers, ) -from sdm_client.result import PredictionResult +from sdm_client.result import Explanations, PredictionResult from sdm_client.schema import _as_key_tuple from sdm_client.wire.keys import ( _named_by, @@ -916,7 +916,10 @@ def _carry_the_answers_already_paid_for( def _numbered_from(result: PredictionResult, start: int) -> PredictionResult: - r"""``result`` with each row numbered ``start`` places further on.""" + r"""``result`` with each row numbered ``start`` places further on. + + An explanation kept under its ``row_index`` moves with its row. + """ if start: result.items = tuple( dataclasses.replace(item, row_index=item.row_index + start) @@ -924,6 +927,7 @@ def _numbered_from(result: PredictionResult, start: int) -> PredictionResult: else item for item in result.items ) + result._explanations = result._explanations._renumbered(start) return result @@ -1256,7 +1260,7 @@ def _warn_of_steps_that_carry_no_explanation( missing = [ step for (_, step), answer in zip(rewritten.parts, answers, strict=True) - if answer.explanation is None + if not answer.explanations ] if not missing or len(missing) == len(answers): return @@ -1282,26 +1286,26 @@ def _by_step( the pair of row and step is what names a point in the series, which is how the contract reads it. - Each step is explained on its own, so the explanations are kept under the - step they belong to. Dropping them left the result saying it carried an - explanation while holding none. + Each step is explained on its own, so each row's explanation is kept + under ``(id, forecast_step)``, the pair that names its point in the frame. """ items = [ dataclasses.replace(item, forecast_step=step) for (_, step), answer in zip(rewritten.parts, answers, strict=True) for item in answer.items ] - by_step = { - step: answer.explanation - for (_, step), answer in zip(rewritten.parts, answers, strict=True) - if answer.explanation is not None - } + explanations = Explanations._merged( + [ + answer.explanations._at_step(step) + for (_, step), answer in zip(rewritten.parts, answers, strict=True) + ] + ) built = PredictionResult( model=answers[0].model if answers else '', request_id=answers[0].request_id if answers else '', items=items, outputs=tuple(answers[0].outputs) if answers else (), - explanation={'by_forecast_step': by_step} if by_step else None, + explanations=explanations, ) # `forecast_step` is put on the items above, so with no items there is # nothing to read it from and an empty forecast lost the column that says diff --git a/packages/structured-data-models-client/src/sdm_client/result.py b/packages/structured-data-models-client/src/sdm_client/result.py index c76c084e4..7c6937239 100644 --- a/packages/structured-data-models-client/src/sdm_client/result.py +++ b/packages/structured-data-models-client/src/sdm_client/result.py @@ -11,7 +11,14 @@ >>> result = client.predict(request, model=KumoRelational(), ... outputs=['prediction', 'explanation']) >>> result.to_pandas() # always a frame ->>> result.explanation # None unless you asked for it +>>> result.explanations # {} unless you asked for them +>>> result.explanations['user_5'] # one row's explanation +>>> result.explanations[('user_5', 3)] # a forecast's, at step 3 +>>> result.explanations.to_pandas() # every gradient, one per line + +Each explained row is under the key that names it in ``to_pandas()``: its +``id``, ``(id, forecast_step)`` for a forecast, and its ``row_index`` when the +request has no entity key. The frame's shape depends on the **task**, not the model. Classification and multiclass both give one row per entity. Multiclass carries a probability per @@ -29,7 +36,7 @@ class as columns rather than a row per class, while forecasting gives one row from __future__ import annotations -from collections.abc import Iterator, Sequence +from collections.abc import Iterator, Mapping, Sequence from typing import Any import pandas as pd @@ -55,6 +62,170 @@ class as columns rather than a row per class, while forecasting gives one row 'embeddings', ) +# The columns of `Explanations.to_pandas()` after the ones naming the row. +_GRADIENT_COLUMNS = ( + 'table', + 'hop', + 'key', + 'row', + 'output', + 'column', + 'gradient', +) + + +class Explanations(dict[Any, dict[str, Any]]): + r"""Each explained row's explanation, under the key naming it in the frame. + + The key is the row's ``id``, ``(id, forecast_step)`` for a forecast, and + its ``row_index`` when the request has no entity key. An entry is a plain + dict:: + + {'outputs': [...], # the explained scores + 'instance': {'columns': [...], + 'gradients': [...]}, # [output][column] + 'related': {table: {'keys': [...], # absent for a keyless table + 'rows': [{...}], # each row's values as sent + 'columns': [...], + 'gradients': [...]}}} # [output][row][column] + + ``related`` is there for ``kumo-relational`` only. ``columns`` are the + model's preprocessed numerical columns, as the endpoint names them. + + Empty when no explanation was asked for, or none came back. + """ + + def __init__( + self, + entries: Mapping[Any, dict[str, Any]] | None = None, + *, + keyed_by: tuple[str, ...] = ('id',), + hops: Mapping[Any, Mapping[str, Sequence[int | None]]] | None = None, + ) -> None: + super().__init__(entries or {}) + # What each part of a key is, which names the frame's first columns. + self._keyed_by = keyed_by + # How many links each related row is from the entity, per entry and + # table, aligned with that table's `rows`. Kept beside the entries + # rather than in them, because an entry is the endpoint's shape. + self._hops = {key: dict(tables) for key, tables in (hops or {}).items()} + + def to_pandas(self) -> pd.DataFrame: + r"""Every gradient, one line each, across all explained rows. + + One line per (row, table, related row, output, column). ``table`` is + ``'instance'`` for the row's own block and the table's name for a + related one. ``hop`` is how many links the row is from the entity: 0 + for the entity itself, and ``None`` where the rows sent do not link + it. ``key`` is the related row's primary key, ``None`` for a keyless + table, and the row's own ``id`` for its instance block. ``row`` is the + position in that entry's ``related[table]['rows']``. ``output`` names + the score as the column of ``result.to_pandas()`` that holds it. + """ + lines: list[tuple[Any, ...]] = [] + for key, entry in self.items(): + named = key if len(self._keyed_by) > 1 else (key,) + outputs = [_output_named(output) for output in entry['outputs']] + instance = entry['instance'] + own = named[0] if self._keyed_by[0] == 'id' else None + for output, by_column in zip( + outputs, instance['gradients'], strict=True + ): + for column, gradient in zip( + instance['columns'], by_column, strict=True + ): + lines.append( + ( + *named, + 'instance', + 0, + own, + 0, + output, + column, + gradient, + ) + ) + hops = self._hops.get(key, {}) + for table, block in entry.get('related', {}).items(): + keys = block.get('keys') or [None] * len(block['rows']) + at = hops.get(table) or [None] * len(block['rows']) + for output, by_row in zip( + outputs, block['gradients'], strict=True + ): + for row, by_column in enumerate(by_row): + for column, gradient in zip( + block['columns'], by_column, strict=True + ): + lines.append( + ( + *named, + table, + at[row], + keys[row], + row, + output, + column, + gradient, + ) + ) + # Built as objects so a key keeps its own type: an integer key beside a + # keyless table's `None` would otherwise come back a float. + return pd.DataFrame( + lines, columns=[*self._keyed_by, *_GRADIENT_COLUMNS], dtype=object + ).astype({'hop': 'Int64', 'row': 'int64', 'gradient': 'float64'}) + + def _renumbered(self, start: int) -> Explanations: + r"""These, with each ``row_index`` key ``start`` places further on. + + A key that is an ``id`` is not a position and is left alone. + """ + if not start or self._keyed_by != ('row_index',): + return self + return Explanations( + {key + start: entry for key, entry in self.items()}, + keyed_by=self._keyed_by, + hops={key + start: hops for key, hops in self._hops.items()}, + ) + + def _at_step(self, step: Any) -> Explanations: + r"""These, as the explanations of one forecast step.""" + return Explanations( + {(key, step): entry for key, entry in self.items()}, + keyed_by=(*self._keyed_by, 'forecast_step'), + hops={(key, step): hops for key, hops in self._hops.items()}, + ) + + @classmethod + def _merged(cls, parts: Sequence[Explanations]) -> Explanations: + r"""The explanations of several calls, as one. + + Each call answers about its own rows, so no key is in two parts. + """ + explained = [part for part in parts if part] + if not explained: + return cls() + merged = cls(keyed_by=explained[0]._keyed_by) + for part in explained: + merged.update(part) + merged._hops.update(part._hops) + return merged + + +def _output_named(output: Mapping[str, Any]) -> str: + r"""The column of ``to_pandas()`` that holds this explained score. + + A probability is ``probability_`` and a quantile + ``quantile_``, as the frame spreads them, so a gradient can be read + beside the score it moves. + """ + kind = output.get('kind') + if kind == 'probability': + return f'probability_{output.get("class")}' + if kind == 'quantile': + return f'quantile_{output.get("level")}' + return str(kind) + class PredictionResult: r"""The answer to one request. @@ -80,7 +251,7 @@ def __init__( request_id: str, items: Sequence[PredictionItem], metadata: dict[str, Any] | None = None, - explanation: dict[str, Any] | None = None, + explanations: Mapping[Any, dict[str, Any]] | None = None, outputs: Sequence[str] = (), unscored_candidates: Sequence[Any] = (), ) -> None: @@ -96,7 +267,11 @@ def __init__( # about. Set where a forecast is assembled, for the same reason. self._names_its_step = False self.metadata = dict(metadata or {}) - self._explanation = explanation + self._explanations = ( + explanations + if isinstance(explanations, Explanations) + else Explanations(explanations) + ) self._unscored_candidates = tuple(unscored_candidates) def absent(self, requested: Sequence[str]) -> tuple[str, ...]: @@ -128,7 +303,7 @@ def _carries(self, name: str) -> bool: was refused before the request was sent. """ if name == 'explanation': - return self._explanation is not None + return bool(self._explanations) if name not in _ITEM_FIELDS: return True return any( @@ -137,12 +312,18 @@ def _carries(self, name: str) -> bool: ) @property - def explanation(self) -> dict[str, Any] | None: - r"""The explanation, when asked for and the model produced one. + def explanations(self) -> Explanations: + r"""Each explained row's explanation, under the key naming that row. + + The key is the one ``to_pandas()`` names the row by: its ``id``, + ``(id, forecast_step)`` for a forecast, and ``row_index`` when the + request has no entity key. Kept out of the frame, because an entry is + nested and the frame is flat. ``explanations.to_pandas()`` flattens + them. - ``None`` otherwise, rather than a different return type. + Empty when not asked for, rather than a different return type. """ - return self._explanation + return self._explanations @property def unscored_candidates(self) -> tuple[Any, ...]: @@ -272,8 +453,8 @@ def merge(cls, parts: Sequence[PredictionResult]) -> PredictionResult: # would otherwise survive. That is the one you do not have when a # later batch is the one that misbehaved. metadata['batch_request_ids'] = [part.request_id for part in parts] - explanation = next( - (part.explanation for part in parts if part.explanation), None + explanations = Explanations._merged( + [part.explanations for part in parts] ) # Every part answers one slice of the same request, so they were asked # for the same fields. If they were not, the merged result cannot @@ -300,7 +481,7 @@ def merge(cls, parts: Sequence[PredictionResult]) -> PredictionResult: request_id=first.request_id, items=items, metadata=metadata, - explanation=explanation, + explanations=explanations, outputs=first.outputs, unscored_candidates=unscored, ) @@ -315,7 +496,7 @@ def __iter__(self) -> Iterator[PredictionItem]: return iter(self.items) def __repr__(self) -> str: - explained = ', explained' if self._explanation is not None else '' + explained = ', explained' if self._explanations else '' left_out = len(self._unscored_candidates) unscored = ( f', {left_out} candidate{"s" if left_out > 1 else ""} unscored' diff --git a/packages/structured-data-models-client/src/sdm_client/wire/explanations.py b/packages/structured-data-models-client/src/sdm_client/wire/explanations.py new file mode 100644 index 000000000..39a31e02d --- /dev/null +++ b/packages/structured-data-models-client/src/sdm_client/wire/explanations.py @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""Reading the explanation each prediction record carries. + +Both models answer an explanation in one format, ``sdm_gradient_v1``, one per +record, about that record's own row. Its blocks name rows by their position in +the tables of the call that produced it, and a request sent in several calls +starts those positions at 0 in each. So each record is read here, beside the +payload of its own call, into an entry that needs no payload to be read: the +row it is about becomes its key, and a related row its key and its values. + +The format is the endpoint's: ``gradients`` is ``[output][row][column]``, and +``columns`` are the model's preprocessed numerical columns, passed through. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from typing import Any + +from sdm_client.errors import StructuredDataError +from sdm_client.result import Explanations + +GRADIENT_FORMAT = 'sdm_gradient_v1' + +#: Reads one record's ``related_tables`` into the entry's ``related``, and +#: says how many links each of those rows is from the entity. +RelatedReader = Callable[ + [Mapping[str, Any]], + tuple[dict[str, Any], dict[str, list[int | None]]], +] + + +def explanations_of( + items: Sequence[Any], related: RelatedReader | None = None +) -> Explanations: + r"""Each answered row's explanation, under the key naming that row. + + ``items`` have their ``id`` restored already, so the key is the one the + frame shows: the ``id``, or the ``row_index`` when the rows carry none. + ``related`` is given for a model that reads related tables; without it an + entry has no ``related`` at all, rather than an empty one. + + Raises: + StructuredDataError: If a record's explanation is in a format this + client cannot read. + """ + explained = [item for item in items if item.explanation] + if not explained: + return Explanations() + by_id = any(item.id is not None for item in explained) + entries: dict[Any, dict[str, Any]] = {} + hops: dict[Any, dict[str, list[int | None]]] = {} + for item in explained: + key = item.id if by_id else item.row_index + details = _details_of(item.explanation) + instance = details.get('instance_table') or {} + entry: dict[str, Any] = { + 'outputs': list(details.get('outputs') or []), + 'instance': { + 'columns': list(instance.get('columns') or []), + # One row, the record's own, so the row axis says nothing. + 'gradients': [ + by_row[0] for by_row in instance.get('gradients') or [] + ], + }, + } + if related is not None: + entry['related'], hops[key] = related( + details.get('related_tables') or {} + ) + entries[key] = entry + return Explanations( + entries, + keyed_by=('id',) if by_id else ('row_index',), + hops=hops, + ) + + +def _details_of(explanation: Mapping[str, Any]) -> Mapping[str, Any]: + r"""The body of an explanation, in the one format this client reads.""" + given = explanation.get('format') + if given != GRADIENT_FORMAT: + raise StructuredDataError( + f'the response carries an explanation in format {given!r}, and ' + f'this client reads {GRADIENT_FORMAT!r} only', + code='INVALID_RESPONSE', + ) + return explanation.get('details') or {} diff --git a/packages/structured-data-models-client/tests/test_envelope_and_result.py b/packages/structured-data-models-client/tests/test_envelope_and_result.py index 8932d3874..c2e5dd99d 100644 --- a/packages/structured-data-models-client/tests/test_envelope_and_result.py +++ b/packages/structured-data-models-client/tests/test_envelope_and_result.py @@ -244,7 +244,7 @@ def test_the_entity_key_is_carried_when_the_model_has_one(): def test_an_explanation_is_absent_rather_than_a_different_return_type(): result = result_with({'row_index': 0, 'prediction': 1}) - assert result.explanation is None + assert result.explanations == {} assert isinstance(result.to_pandas(), pd.DataFrame) @@ -325,7 +325,7 @@ def forecast_through_the_client(explanations, requested): PredictionItem(row_index=0, prediction=1.0, forecast_step=None) ], outputs=tuple(requested), - explanation=held, + explanations=held, ) for held in explanations ] @@ -346,7 +346,7 @@ def test_a_forecast_step_that_carries_no_explanation_is_named(): answer carries an explanation sees the step that did and says nothing about the one that did not.""" said = forecast_through_the_client( - [{'cohorts': ['a']}, None], ['prediction', 'explanation'] + [{0: {'outputs': []}}, None], ['prediction', 'explanation'] ) assert any('step 2 carries none' in message for message in said) @@ -354,7 +354,8 @@ def test_a_forecast_step_that_carries_no_explanation_is_named(): def test_a_forecast_whose_steps_all_explain_is_not_reported(): said = forecast_through_the_client( - [{'cohorts': ['a']}, {'cohorts': ['b']}], ['prediction', 'explanation'] + [{0: {'outputs': []}}, {0: {'outputs': []}}], + ['prediction', 'explanation'], ) assert not any('carries none' in message for message in said) @@ -362,7 +363,7 @@ def test_a_forecast_whose_steps_all_explain_is_not_reported(): def test_a_forecast_that_was_not_asked_for_an_explanation_is_not_reported(): said = forecast_through_the_client( - [{'cohorts': ['a']}, None], ['prediction'] + [{0: {'outputs': []}}, None], ['prediction'] ) assert said == [] diff --git a/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py b/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py index d2e1fcc40..43f5b137e 100644 --- a/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py +++ b/packages/structured-data-models-client/tests/test_limits_and_absent_outputs.py @@ -130,13 +130,13 @@ def __init__(self, **fields): setattr(self, name, value) -def result(outputs=(), explanation=None, **fields) -> PredictionResult: +def result(outputs=(), explanations=None, **fields) -> PredictionResult: return PredictionResult( model='kumo-tabular', request_id='r1', items=[FakeItem(row_index=0, **fields)], outputs=outputs, - explanation=explanation, + explanations=explanations, ) @@ -167,14 +167,14 @@ def test_a_repeated_request_is_reported_once(): def test_explanation_is_looked_for_where_it_lives(): assert result(prediction=1.0).absent(['explanation']) == ('explanation',) - assert result(prediction=1.0, explanation=None).absent(['explanation']) == ( + assert result(prediction=1.0, explanations={}).absent(['explanation']) == ( 'explanation', ) carried = PredictionResult( model='m', request_id='r', items=[FakeItem(row_index=0)], - explanation={'feature_importance': {}}, + explanations={0: {'outputs': []}}, ) assert carried.absent(['explanation']) == () diff --git a/packages/structured-data-models-client/tests/test_tabular_explanations.py b/packages/structured-data-models-client/tests/test_tabular_explanations.py index 743af6bbc..af3b1a82a 100644 --- a/packages/structured-data-models-client/tests/test_tabular_explanations.py +++ b/packages/structured-data-models-client/tests/test_tabular_explanations.py @@ -4,8 +4,8 @@ """Asking ``kumo-tabular`` for an explanation. -The endpoint explains one scored row at a time, so the client refuses more -before sending, and hands the explanation it gets back to the caller. +The endpoint explains every scored row, and the client hands each row's +explanation back under the key naming that row. """ from __future__ import annotations @@ -82,23 +82,40 @@ def test_session_rows_sent(tabular_request): def test_result_carries_explanation(): - """The explanation in the response reaches the result unchanged. + """The row's explanation reaches the result under its row_index. Assumptions: - - The endpoint returns one record with an explanation dictionary. + - The endpoint returns one record, with no id, and its explanation. """ - explanation = {'format': 'sdm_gradient_v1', 'details': {'outputs': []}} + explanation = { + 'format': 'sdm_gradient_v1', + 'details': { + 'outputs': [{'kind': 'prediction', 'statistic': 'mean'}], + 'instance_table': { + 'row_indices': [0], + 'columns': ['x'], + 'gradients': [[[0.5]]], + }, + 'related_tables': {}, + }, + } result = TabularAdapter().normalize( response(explanation=explanation), EXPLAIN ) - assert result.explanation == explanation + assert result.explanations == { + 0: { + 'outputs': [{'kind': 'prediction', 'statistic': 'mean'}], + 'instance': {'columns': ['x'], 'gradients': [[0.5]]}, + } + } def test_result_without_explanation(): - """A response without an explanation leaves the result's explanation None. + """A response without an explanation leaves the result's explanations + empty. Assumptions: - The record carries no explanation field. """ result = TabularAdapter().normalize(response(), ('prediction',)) - assert result.explanation is None + assert result.explanations == {} diff --git a/tests/golden/public_api.json b/tests/golden/public_api.json index 657af9887..eb77f0a0d 100644 --- a/tests/golden/public_api.json +++ b/tests/golden/public_api.json @@ -112,6 +112,7 @@ "sdm_client": { "__all__": [ "Column", + "Explanations", "FittedModel", "KumoRelational", "KumoTabular", @@ -149,6 +150,17 @@ }, "signature": "(name: 'str', dtype: 'str', stype: 'str') -> None" }, + "Explanations": { + "bases": [ + "builtins.dict" + ], + "kind": "class", + "public_attributes": [], + "public_methods": { + "to_pandas": "(self) -> 'pd.DataFrame'" + }, + "signature": "(entries: 'Mapping[Any, dict[str, Any]] | None' = None, *, keyed_by: 'tuple[str, ...]' = ('id',), hops: 'Mapping[Any, Mapping[str, Sequence[int | None]]] | None' = None) -> 'None'" + }, "FittedModel": { "bases": [], "kind": "class", @@ -226,7 +238,7 @@ "bases": [], "kind": "class", "public_attributes": [ - "explanation", + "explanations", "unscored_candidates" ], "public_methods": { @@ -234,7 +246,7 @@ "merge": "(parts: 'Sequence[PredictionResult]') -> 'PredictionResult'", "to_pandas": "(self) -> 'pd.DataFrame'" }, - "signature": "(*, model: 'str', request_id: 'str', items: 'Sequence[PredictionItem]', metadata: 'dict[str, Any] | None' = None, explanation: 'dict[str, Any] | None' = None, outputs: 'Sequence[str]' = (), unscored_candidates: 'Sequence[Any]' = ()) -> 'None'" + "signature": "(*, model: 'str', request_id: 'str', items: 'Sequence[PredictionItem]', metadata: 'dict[str, Any] | None' = None, explanations: 'Mapping[Any, dict[str, Any]] | None' = None, outputs: 'Sequence[str]' = (), unscored_candidates: 'Sequence[Any]' = ()) -> 'None'" }, "PredictiveQuery": { "bases": [], From aa2edea88a65cf2023aba6a6ecddc12bdc4d9b41 Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Thu, 29 Oct 2026 13:17:20 -0700 Subject: [PATCH 06/25] Read an explanation's details without checking its format The client refused an explanation whose format was not sdm_gradient_v1 with INVALID_RESPONSE. Both models answer in that one format, so the check guarded nothing; the details are now read directly. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Jana Gagacheva --- .../src/sdm_client/wire/explanations.py | 20 +------------------ 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/packages/structured-data-models-client/src/sdm_client/wire/explanations.py b/packages/structured-data-models-client/src/sdm_client/wire/explanations.py index 39a31e02d..54155cd9e 100644 --- a/packages/structured-data-models-client/src/sdm_client/wire/explanations.py +++ b/packages/structured-data-models-client/src/sdm_client/wire/explanations.py @@ -20,11 +20,8 @@ from collections.abc import Callable, Mapping, Sequence from typing import Any -from sdm_client.errors import StructuredDataError from sdm_client.result import Explanations -GRADIENT_FORMAT = 'sdm_gradient_v1' - #: Reads one record's ``related_tables`` into the entry's ``related``, and #: says how many links each of those rows is from the entity. RelatedReader = Callable[ @@ -42,10 +39,6 @@ def explanations_of( frame shows: the ``id``, or the ``row_index`` when the rows carry none. ``related`` is given for a model that reads related tables; without it an entry has no ``related`` at all, rather than an empty one. - - Raises: - StructuredDataError: If a record's explanation is in a format this - client cannot read. """ explained = [item for item in items if item.explanation] if not explained: @@ -55,7 +48,7 @@ def explanations_of( hops: dict[Any, dict[str, list[int | None]]] = {} for item in explained: key = item.id if by_id else item.row_index - details = _details_of(item.explanation) + details = item.explanation.get('details') or {} instance = details.get('instance_table') or {} entry: dict[str, Any] = { 'outputs': list(details.get('outputs') or []), @@ -78,14 +71,3 @@ def explanations_of( hops=hops, ) - -def _details_of(explanation: Mapping[str, Any]) -> Mapping[str, Any]: - r"""The body of an explanation, in the one format this client reads.""" - given = explanation.get('format') - if given != GRADIENT_FORMAT: - raise StructuredDataError( - f'the response carries an explanation in format {given!r}, and ' - f'this client reads {GRADIENT_FORMAT!r} only', - code='INVALID_RESPONSE', - ) - return explanation.get('details') or {} From 964ded9933bc2ad57118a04187d03c73726edc55 Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Thu, 29 Oct 2026 13:18:28 -0700 Subject: [PATCH 07/25] Correct the relational explanation docs against the code kumo-relational sends the entity as two tables: the per-prediction instance row (entity key, anchor time, engine bookkeeping) and the entity's own feature row as a related table. The docs put the entity's features under instance; they are under related['users'], and to_pandas() shows the entity on an instance line and a users line, both hop 0. The example tables now name each output by its result column (probability_yes), and say what key and hop hold on an instance line and for an unreached row. The relational probe for an explanation over several entities expected a refusal before the wire; it now checks one entry per scored entity, keyed by id, with related tables. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Jana Gagacheva --- docs/about/architecture.md | 9 ++++ docs/reference/prediction-output.md | 48 ++++++++++++----- qa/probe_relational.py | 51 +++++++++++++++++-- skills/relational/SKILL.md | 11 ++-- .../references/beyond-one-answer.md | 11 ++-- 5 files changed, 105 insertions(+), 25 deletions(-) diff --git a/docs/about/architecture.md b/docs/about/architecture.md index 9c9be1fb1..7376c42ab 100644 --- a/docs/about/architecture.md +++ b/docs/about/architecture.md @@ -611,6 +611,15 @@ it on the result. `result.explanations.to_pandas()` flattens them into a table of their own when one is wanted, with each related row's `hop` from the entity worked out from the relationships and foreign-key values the client sent. +**The relational entity is two tables.** The engine sends the per-prediction +row, the entity key, anchor time and its own bookkeeping, as the instance +table, and the entity's own feature row as a related table sampled like any +other. So a `kumo-relational` entry's `instance` holds the bookkeeping row's +gradients, and the entity's features are under `related`, by their table's +name. The flat table shows the entity on an `instance` line and on a line of +its own table, both hop 0. A `kumo-tabular` entry has one table, the +`instance`. + A forecast is explained a step at a time, under `(id, forecast_step)`, rather than aggregated across steps: each step is its own row in `to_pandas()`, and its explanation is keyed the same way. diff --git a/docs/reference/prediction-output.md b/docs/reference/prediction-output.md index e03a4a068..cf824df6b 100644 --- a/docs/reference/prediction-output.md +++ b/docs/reference/prediction-output.md @@ -192,7 +192,13 @@ table without numerical features has an empty `columns`. | kumo-tabular | the one table: each predicted row's own columns | | kumo-relational | the predicted entity's own table, plus every related table in the subgraph the client gathered for that entity (all tables reached within the hop limit, up to 6) | -A `kumo-tabular` entry has no `related` key. +A `kumo-tabular` entry has no `related` key: its one table is the `instance`. + +`kumo-relational` sends the entity as two tables. `instance` is the +per-prediction row the engine builds: the entity key, the anchor time and the +engine's own bookkeeping, with few or no numerical columns. The entity's own +feature row is a related table like any other, under its table's name, such +as `related['users']`, and that is where its features' gradients are. Only the rows being predicted and their related rows get gradients. The context rows, the labelled examples the model compares against, get none. @@ -222,17 +228,23 @@ result.explanations['cust_17'] # 'gradients': [[0.01, -0.03], [-0.01, 0.03]]}} ``` -`kumo-relational`, user 5 with one order and two login events. `orders` has a -primary key (`order_id`), so its rows carry `keys`. `events` has none, so its -rows are identified by their values alone. Both are part of the same entry: +`kumo-relational`, user 5 with one order and two login events. The user's own +features are under `related['users']`; `instance` holds the engine's +per-prediction row. `orders` has a primary key (`order_id`), so its rows carry +`keys`. `events` has none, so its rows are identified by their values alone. +All are part of the same entry: ```python result.explanations['user_5'] # {'outputs': [{'kind': 'probability', 'class': 'no'}, # {'kind': 'probability', 'class': 'yes'}], -# 'instance': {'columns': ['tenure_days'], -# 'gradients': [[-0.05], [0.05]]}, +# 'instance': {'columns': ['anchor_time_dayofweek'], # engine bookkeeping, illustrative +# 'gradients': [[0.001], [-0.001]]}, # 'related': { +# 'users': {'keys': ['user_5'], +# 'rows': [{'user_id': 'user_5', 'tenure_days': 410}], +# 'columns': ['tenure_days'], +# 'gradients': [[[-0.05]], [[0.05]]]}, # 'orders': {'keys': ['order_9003'], # 'rows': [{'user_id': 'user_5', 'order_id': 'order_9003', 'amount': 12.0}], # 'columns': ['amount'], @@ -250,15 +262,23 @@ result.explanations['user_5'] across all entities: ``` -id table hop key row output column gradient -user_5 users 0 user_5 0 P(yes) tenure_days 0.05 -user_5 orders 1 order_9003 0 P(yes) amount 0.02 -user_5 products 2 prod_77 0 P(yes) price 0.004 -user_5 events 1 - 1 P(yes) kind_login 0.02 +id table hop key row output column gradient +user_5 instance 0 user_5 0 probability_yes anchor_time_dayofweek -0.001 +user_5 users 0 user_5 0 probability_yes tenure_days 0.05 +user_5 orders 1 order_9003 0 probability_yes amount 0.02 +user_5 products 2 prod_77 0 probability_yes price 0.004 +user_5 events 1 None 1 probability_yes kind_login -0.02 ``` +- `table` is `instance` for the entry's `instance` block and the table's name + for a related one. On `kumo-relational` the entity shows on both an + `instance` line and a line of its own table (`users` here), both hop 0. - `hop` is how many links the row is from the entity, 0 for the entity itself. The client works it out from the relationships and foreign-key - values it sent. -- `key` is the row's primary key, or empty for a keyless table. `row` is the - row's position in that entity's `related[table]['rows']`. + values it sent. It is `None` for a related row no relationship reaches. +- `key` is the row's primary key, or `None` for a keyless table. On an + `instance` line it is the row's `id`, or `None` when the request has no + entity key. `row` is the row's position in that entity's + `related[table]['rows']`, and 0 on an `instance` line. +- `output` names the score by the column of `result.to_pandas()` that holds + it: `probability_yes`, `prediction`, `quantile_0.1`. diff --git a/qa/probe_relational.py b/qa/probe_relational.py index 4b9141f70..b9c9b58a0 100644 --- a/qa/probe_relational.py +++ b/qa/probe_relational.py @@ -428,21 +428,64 @@ def rows(request): note(e, message[:160]) with probe( - AREA, 'an explanation over several entities', 'refused before the wire' + AREA, + 'an explanation over several entities', + 'one entry per scored entity, keyed by id, with related tables, ' + 'or refused for a stated reason', ) as e: built = schema() request = PredictiveQuery( 'PREDICT Orders.ShipVia FOR EACH Orders.OrderID', built ).materialize(indices=order_ids(built, 4), num_context_samples=200) try: - client.predict( + result = client.predict( request, model=KumoRelational(), outputs=['prediction', 'explanation'], ) - fail(e, 'four entities were accepted for an explanation') except StructuredDataError as exc: - note(e, str(exc)[:140]) + note(e, f'refused: {str(exc)[:160]}') + else: + frame = result.to_pandas() + explained = result.explanations + if set(explained) != set(frame['id']): + fail( + e, + f'explained {sorted(explained)}, ' + f'scored {sorted(frame["id"])}', + ) + elif not all(entry.get('related') for entry in explained.values()): + fail(e, 'an entry carries no related tables') + elif any( + len(entry['instance']['gradients']) != len(entry['outputs']) + or any( + len(block['gradients']) != len(entry['outputs']) + or any( + len(per_output) != len(block['rows']) + for per_output in block['gradients'] + ) + for block in entry['related'].values() + ) + for entry in explained.values() + ): + fail( + e, + 'gradients are not [output][column] and [output][row][column]', + ) + else: + flat = explained.to_pandas() + if 'explanation' in frame.columns: + fail(e, 'the explanation leaked into to_pandas()') + elif not set(flat['id']) <= set(explained): + fail(e, f'to_pandas() ids: {sorted(set(flat["id"]))}') + else: + first = explained[min(explained)] + note( + e, + f'{len(explained)} entries, ' + f'related {sorted(first["related"])}, ' + f'{len(flat)} lines flat', + ) with probe( AREA, diff --git a/skills/relational/SKILL.md b/skills/relational/SKILL.md index 7ad3b0c36..28d2526b9 100644 --- a/skills/relational/SKILL.md +++ b/skills/relational/SKILL.md @@ -191,17 +191,20 @@ of its own, and they are in An explanation is kept out of `to_pandas()`, on `result.explanations`, a mapping from each row's key to its explanation: the entity's `id`, or -`(id, forecast_step)` on a forecast. One entry covers the entity's own row and -every related row gathered for it: +`(id, forecast_step)` on a forecast. One entry covers the engine's +per-prediction row, the entity's own feature row and every related row +gathered for it. The entity's features are under its table's name in +`related`, not in `instance`: ```python entry = result.explanations['user_5'] # the entity whose id is 'user_5' -entry['instance']['gradients'] # [output][column], the entity's own row +entry['instance']['gradients'] # [output][column], the per-prediction row: key, anchor time, bookkeeping +entry['related']['users'] # the entity's own features, one row orders = entry['related']['orders'] # one block per related table orders['keys'] # each row's primary key; absent if the table has none orders['rows'] # each row's values as sent orders['gradients'] # [output][row][column] -result.explanations.to_pandas() # one line per (row, output, column), with its hop +result.explanations.to_pandas() # one line per (row, output, column), with its hop; the entity is hop 0 on both its instance and users lines ``` ## Asking the same context more than once diff --git a/skills/relational/references/beyond-one-answer.md b/skills/relational/references/beyond-one-answer.md index d20ef6c00..38b340731 100644 --- a/skills/relational/references/beyond-one-answer.md +++ b/skills/relational/references/beyond-one-answer.md @@ -120,8 +120,12 @@ entity's `id`, or `(id, forecast_step)` on a forecast, since a forecast is keyed by the pair. It is kept out of `to_pandas()` itself. An entry holds `outputs`, the explained scores in order; `instance`, the -entity's own row, with `columns` and `gradients` as `[output][column]`; and -`related`, one block per related table the client gathered for that entity. +engine's per-prediction row (the entity key, anchor time and its own +bookkeeping, with few or no numerical columns), with `columns` and `gradients` +as `[output][column]`; and `related`, one block per related table the client +gathered for that entity. The entity's own feature row is one of those blocks, +under its table's name, such as `related['users']`, so its features' gradients +are there and not in `instance`. A block carries `keys`, each row's primary key, when the table has one, and always `rows`, each row's values as sent, with `columns` and `gradients` as `[output][row][column]`, aligned with those rows. A keyless table's rows are @@ -132,7 +136,8 @@ datetimes may appear as derived columns, and a table without numerical features gives an empty list. The context rows get no gradients. A row reached through several paths appears once, with its total effect through all of them. `result.explanations.to_pandas()` flattens every entry into one line per (row, -output, column), with `hop`, how many links the row is from the entity. +output, column), with `hop`, how many links the row is from the entity. The +entity shows on an `instance` line and on a line of its own table, both hop 0. A ranking refuses an explanation locally, and the shipping endpoint refuses it everywhere else, so treat `result.explanations` as possibly empty. A deployment From a79c00aa09f5fe439d153dbc257c0f5d557bee91 Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Thu, 29 Oct 2026 13:25:39 -0700 Subject: [PATCH 08/25] Test per-row explanations against mocked NIM answers Tabular: every scored row is explained under its id, or its row_index when the request has no key; the instance gradients are the row's own, [output][column]; an entry has no related key; outputs and columns are kept as returned. Relational, on payloads shaped as the engine gathers them and answers shaped as the NIM user guide gives them: related positions are read as that call's rows, a keyed table's keys come back as their declared dtype (a key past 2**53 included) while rows keep the values as sent, a keyless (__node_id) table has no keys, and gradients line up with rows. A request split into calls that each number from 0 keeps every entity's own rows, and row_index keys move with their rows. A forecast is keyed (id, forecast_step). Explanations.to_pandas(): its columns, one line per (row, table, related row, output, column), hop 0/1/2 through a two-hop graph and empty for an unlinked row, and outputs named as the frame's columns. The NIM explanation builder is a shared conftest fixture, and tabular_request can build a request with no key column. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Jana Gagacheva --- .../tests/conftest.py | 56 +- .../tests/test_relational_explanations.py | 671 ++++++++++++++++++ .../tests/test_tabular_explanations.py | 268 ++++++- 3 files changed, 980 insertions(+), 15 deletions(-) create mode 100644 packages/structured-data-models-client/tests/test_relational_explanations.py diff --git a/packages/structured-data-models-client/tests/conftest.py b/packages/structured-data-models-client/tests/conftest.py index 652c4b081..431a850ff 100644 --- a/packages/structured-data-models-client/tests/conftest.py +++ b/packages/structured-data-models-client/tests/conftest.py @@ -10,7 +10,8 @@ from __future__ import annotations -from collections.abc import Callable +from collections.abc import Callable, Mapping, Sequence +from typing import Any import pandas as pd import pytest @@ -45,9 +46,12 @@ def tabular_request() -> Callable[..., Request]: """Build a single-table request: 20 labelled context rows, ``rows`` to score. A classification target alternates 0/1; a regression target is a float. + ``keyed=False`` drops the ``row_id`` column, so the rows have no key. """ - def build(rows: int = 1, task: str = 'classification') -> Request: + def build( + rows: int = 1, task: str = 'classification', keyed: bool = True + ) -> Request: labels = [ i % 2 if task == 'classification' else float(i) for i in range(20) ] @@ -64,8 +68,56 @@ def build(rows: int = 1, task: str = 'classification') -> Request: 'age': [33.0 + i for i in range(rows)], } ) + if not keyed: + context = context.drop(columns='row_id') + predict = predict.drop(columns='row_id') return Request( context=context, predict=predict, target='churn', task=task ) return build + + +@pytest.fixture +def gradient_explanation() -> Callable[..., dict[str, Any]]: + """Build one prediction record's ``explanation``, as a NIM sends it. + + The ``sdm_gradient_v1`` shape of the NIM user guides: ``row`` is the + record's position in the predict table, ``gradients`` its own + ``[output][column]`` (sent as ``[output][row][column]`` with one row), and + ``related`` maps a table to ``(row_indices, columns, gradients)`` with + gradients ``[output][row][column]``. + """ + + def build( + *, + outputs: Sequence[Mapping[str, Any]], + row: int, + columns: Sequence[str], + gradients: Sequence[Sequence[float]], + related: Mapping[str, tuple[list[int], list[str], list]] | None = None, + ) -> dict[str, Any]: + return { + 'format': 'sdm_gradient_v1', + 'details': { + 'input_space': 'preprocessed_numerical', + 'outputs': [dict(output) for output in outputs], + 'instance_table': { + 'row_indices': [row], + 'columns': list(columns), + 'gradients': [[list(by_column)] for by_column in gradients], + }, + 'related_tables': { + name: { + 'row_indices': list(indices), + 'columns': list(names), + 'gradients': values, + } + for name, (indices, names, values) in ( + related or {} + ).items() + }, + }, + } + + return build diff --git a/packages/structured-data-models-client/tests/test_relational_explanations.py b/packages/structured-data-models-client/tests/test_relational_explanations.py new file mode 100644 index 000000000..48c2b229a --- /dev/null +++ b/packages/structured-data-models-client/tests/test_relational_explanations.py @@ -0,0 +1,671 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Reading ``kumo-relational`` explanations back as the caller's rows. + +The NIM names each related row by its position in the table as that call sent +it. These payloads are shaped as the engine gathers one (an instance table +carrying ``instance_id`` and the entity reference, related tables keyed by +``instance_id`` and their own key, or the made-up ``__node_id`` when a table +has none), and the answers as the NIM user guide gives them, so what is held +here is that each entity gets back its own rows, keys and gradients. + +The graph per entity: ``users`` (the entity's own row), ``orders`` linked to +it, ``products`` linked to those orders, and ``events``, which has no key. +""" + +from __future__ import annotations + +import warnings +from types import SimpleNamespace +from typing import Any + +import pytest + +from sdm_client import KumoRelational, Request, Schema, SDMClient +from sdm_client.adapters.relational import RelationalAdapter +from sdm_client.core.backend import NimBackend + +EXPLAIN = ('prediction', 'probabilities', 'explanation') +CLASSES = [ + {'kind': 'probability', 'class': 'no'}, + {'kind': 'probability', 'class': 'yes'}, +] +#: Past 2**53, so the contract sends it as base-10 text. +WIDE = 9_007_199_254_740_993 + +#: Each related table as the engine sends it: its columns, and the one +#: numerical column the mocked NIM explains it by. +TABLES = { + 'users': (['instance_id', 'user_id', 'tenure_days'], 'tenure_days'), + 'orders': ( + ['instance_id', 'order_id', 'user_id', 'product_id', 'amount'], + 'amount', + ), + 'products': (['instance_id', 'product_id', 'price'], 'price'), + 'events': ( + ['instance_id', '__node_id', 'user_id', 'kind', 'seconds'], + 'seconds', + ), +} + +#: Each entity's rows, without ``instance_id``. Both users bought product 77, +#: so it is sampled once per entity; product 88 is sampled for user 5 and no +#: order of theirs names it. +ROWS = { + 5: { + 'users': [[5, 30.0]], + 'orders': [[9003, 5, 77, 12.0], [9004, 5, 77, 13.0]], + 'products': [[77, 4.0], [88, 6.0]], + 'events': [[5, 'login', 41], [5, 'login', 42]], + }, + 7: { + 'users': [[7, 70.0]], + 'orders': [[str(WIDE), 7, 77, 17.0]], + 'products': [[77, 4.5]], + 'events': [[7, 'logout', 71]], + }, +} + +RELATIONSHIPS = [ + { + 'source_columns': ['__kumo_entity_ref_0'], + 'target_table': 'users', + 'target_columns': ['user_id'], + }, + { + 'source_table': 'orders', + 'source_columns': ['user_id'], + 'target_table': 'users', + 'target_columns': ['user_id'], + }, + { + 'source_table': 'orders', + 'source_columns': ['product_id'], + 'target_table': 'products', + 'target_columns': ['product_id'], + }, + { + 'source_table': 'events', + 'source_columns': ['user_id'], + 'target_table': 'users', + 'target_columns': ['user_id'], + }, +] + +DTYPES = { + 'instance_id': 'int64', + 'user_id': 'int64', + 'order_id': 'int64', + 'product_id': 'int64', + '__node_id': 'int64', + 'kind': 'string', + 'seconds': 'int64', +} + +PRIMARY_KEYS = { + 'users': ['instance_id', 'user_id'], + 'orders': ['instance_id', 'order_id'], + 'products': ['instance_id', 'product_id'], + 'events': ['instance_id', '__node_id'], +} + + +def gathered(*users: int, first: int = 100, task: str = 'classification'): + """One call's payload, scoring ``users`` as instances ``first`` on. + + Related rows are interleaved across the entities, so a row's position is + never its entity's: reading one by position in the wrong table, or the + wrong call, gives another entity's row. + """ + instances = {user: first + at for at, user in enumerate(users)} + related: dict[str, dict[str, Any]] = {} + for name, (columns, _) in TABLES.items(): + rows: list[list[Any]] = [] + longest = max(len(ROWS[user][name]) for user in users) + for at in range(longest): + for user in users: + if at < len(ROWS[user][name]): + rows.append([instances[user], *ROWS[user][name][at]]) + if name == 'events': + rows = [ + [row[0], position, *row[1:]] + for position, row in enumerate(rows) + ] + related[name] = {'format': 'arrays', 'columns': columns, 'rows': rows} + return { + 'task': { + 'kind': task, + 'target': { + 'column_name': 'TARGET', + 'dtype': 'string' if task != 'regression' else 'float64', + }, + }, + 'schema': { + 'instance_table': { + 'columns': { + 'instance_id': {'dtype': 'int64'}, + '__kumo_entity_ref_0': {'dtype': 'int64'}, + 'TARGET': {'dtype': 'string'}, + }, + 'primary_key': 'instance_id', + }, + 'related_tables': { + name: { + 'columns': { + column: {'dtype': DTYPES.get(column, 'float64')} + for column in columns + }, + 'primary_key': PRIMARY_KEYS[name], + } + for name, (columns, _) in TABLES.items() + }, + 'relationships': RELATIONSHIPS, + }, + 'context': { + 'instance_table': { + 'format': 'arrays', + 'columns': ['instance_id', '__kumo_entity_ref_0', 'TARGET'], + 'rows': [[0, 1, 'no'], [1, 2, 'yes'], [2, 3, 'no']], + }, + }, + 'predict': { + 'instance_table': { + 'format': 'arrays', + 'columns': ['instance_id', '__kumo_entity_ref_0'], + 'rows': [[instances[user], user] for user in users], + }, + 'related_tables': related, + }, + } + + +def nim_answer(payload: dict, gradient_explanation) -> dict: + """What the NIM answers ``payload`` with, explanations included. + + Each record names its related rows by position in this call's tables, + starting at 0, and explains each one by its numerical column ``v`` as + ``[-v, v]`` per class. The instance block's gradient is the entity's + ``row_index + 1``. + """ + predict = payload['predict'] + records = [] + for index, (instance, _) in enumerate(predict['instance_table']['rows']): + related = {} + for name, (columns, explained) in TABLES.items(): + rows = predict['related_tables'][name]['rows'] + mine = [at for at, row in enumerate(rows) if row[0] == instance] + values = [rows[at][columns.index(explained)] for at in mine] + related[name] = ( + mine, + [explained], + [[[-value] for value in values], [[value] for value in values]], + ) + records.append( + { + 'id': str(instance), + 'row_index': index, + 'prediction': 'yes', + 'probabilities': {'no': 0.4, 'yes': 0.6}, + 'explanation': gradient_explanation( + outputs=CLASSES, + row=index, + columns=['anchor_dayofweek'], + gradients=[[-(index + 1.0)], [index + 1.0]], + related=related, + ), + } + ) + return { + 'id': 'r', + 'model': 'kumo-relational', + 'predictions': records, + 'metadata': {}, + } + + +@pytest.fixture +def explained(gradient_explanation): + """Users 5 and 7, scored in one call and explained.""" + payload = gathered(5, 7) + return RelationalAdapter().normalize( + nim_answer(payload, gradient_explanation), EXPLAIN, payload + ) + + +class Transport: + """Answers each call as the NIM would, recording what was sent.""" + + def __init__(self, gradient_explanation) -> None: + self.build = gradient_explanation + self.sent: list[dict] = [] + + def predict(self, payload: dict) -> dict: + self.sent.append(payload) + return nim_answer(payload, self.build) + + def close(self) -> None: + return None + + +def client_answering(gradient_explanation) -> tuple[SDMClient, Transport]: + client = SDMClient(url='http://localhost:8000') + transport = Transport(gradient_explanation) + client._backend = NimBackend(transport) + return client, transport + + +def split(*calls: tuple[int, ...]) -> Request: + """One request gathered as several calls, each numbered from 0.""" + return Request.materialized( + payloads=tuple(gathered(*users) for users in calls), + schema=Schema.from_data({'users': _users_frame()}), + query='PREDICT users.churn FOR EACH users.user_id', + indices=[user for users in calls for user in users], + entity_table='users', + ) + + +def _users_frame(): + import pandas as pd + + return pd.DataFrame({'user_id': [5, 7], 'churn': ['no', 'yes']}) + + +# --- each entity, under its own key ----------------------------------------- + + +def test_every_entity_is_explained_under_its_id(explained): + """Each scored entity's explanation is under its id, as the frame names it. + + Assumptions: + - The call scores users 5 and 7, and the NIM explains both. + """ + assert sorted(explained.explanations) == [5, 7] + assert sorted(explained.explanations) == sorted(explained.to_pandas()['id']) + + +def test_an_entity_has_its_own_instance_gradients(explained): + """The instance block is the record's own, as ``[output][column]``. + + Assumptions: + - The NIM gives the record at row ``i`` the gradients ``-(i + 1)`` and + ``i + 1``. + """ + assert explained.explanations[5]['instance'] == { + 'columns': ['anchor_dayofweek'], + 'gradients': [[-1.0], [1.0]], + } + assert explained.explanations[7]['instance']['gradients'] == [ + [-2.0], + [2.0], + ] + + +def test_an_entity_has_a_block_for_every_related_table(explained): + """Every related table sent is in the entry, under its own name. + + Assumptions: + - The NIM sends one block per related table for every record. + """ + for entry in explained.explanations.values(): + assert set(entry['related']) == set(TABLES) + + +def test_related_rows_are_the_entitys_own_values_as_sent(explained): + """A related row comes back as the values sent, less the engine's + ``instance_id`` and ``__node_id``. + + Assumptions: + - The rows of the two entities are interleaved in every table, so a + position read as the wrong row names the other entity's. + """ + related = explained.explanations[5]['related'] + + assert related['users']['rows'] == [{'user_id': 5, 'tenure_days': 30.0}] + assert related['orders']['rows'] == [ + {'order_id': 9003, 'user_id': 5, 'product_id': 77, 'amount': 12.0}, + {'order_id': 9004, 'user_id': 5, 'product_id': 77, 'amount': 13.0}, + ] + assert related['events']['rows'] == [ + {'user_id': 5, 'kind': 'login', 'seconds': 41}, + {'user_id': 5, 'kind': 'login', 'seconds': 42}, + ] + assert explained.explanations[7]['related']['products']['rows'] == [ + {'product_id': 77, 'price': 4.5} + ] + + +def test_a_keyed_table_gives_each_row_its_key_as_declared(explained): + """A table with a key of its own gives each row that key, read back as its + declared dtype, while ``rows`` keep the value as sent. + + Assumptions: + - ``orders.order_id`` is declared int64, and user 7's order id is past + 2**53, so it travels as text. + """ + orders_7 = explained.explanations[7]['related']['orders'] + + assert explained.explanations[5]['related']['orders']['keys'] == [ + 9003, + 9004, + ] + assert orders_7['keys'] == [WIDE] + assert isinstance(orders_7['keys'][0], int) + assert orders_7['rows'][0]['order_id'] == str(WIDE) + assert explained.explanations[5]['related']['users']['keys'] == [5] + + +def test_a_keyless_table_gives_rows_without_keys(explained): + """A table keyed only by the engine's ``__node_id`` has no ``keys``. + + Assumptions: + - ``events`` has no key of its own; the engine numbers its rows. + """ + events = explained.explanations[5]['related']['events'] + + assert 'keys' not in events + assert all('__node_id' not in row for row in events['rows']) + + +def test_related_gradients_are_aligned_with_rows(explained): + """A related row's gradients sit at its position in ``rows``, as + ``[output][row][column]``. + + Assumptions: + - The NIM explains each related row by its numerical column ``v`` as + ``[-v, v]``, so a gradient names the row it belongs to. + """ + for entry in explained.explanations.values(): + for name, block in entry['related'].items(): + column = TABLES[name][1] + values = [row[column] for row in block['rows']] + assert block['columns'] == [column] + assert block['gradients'] == [ + [[-value] for value in values], + [[value] for value in values], + ] + + +def test_outputs_and_columns_are_passed_through(explained): + """``outputs`` and every ``columns`` are kept as the NIM returned them. + + Assumptions: + - The NIM names its outputs and preprocessed columns; the client does not + read or rename them. + """ + entry = explained.explanations[5] + + assert entry['outputs'] == CLASSES + assert entry['instance']['columns'] == ['anchor_dayofweek'] + assert entry['related']['events']['columns'] == ['seconds'] + + +def test_no_explanation_asked_for_leaves_explanations_empty( + gradient_explanation, +): + """Without ``'explanation'`` in the outputs, ``result.explanations`` is an + empty dict. + + Assumptions: + - The NIM answers without an explanation when none is asked for. + """ + payload = gathered(5, 7) + answer = nim_answer(payload, gradient_explanation) + for record in answer['predictions']: + del record['explanation'] + + result = RelationalAdapter().normalize(answer, ('prediction',), payload) + + assert result.explanations == {} + + +# --- a request sent in several calls ---------------------------------------- + + +def test_a_split_request_reads_each_call_against_its_own_rows( + gradient_explanation, +): + """A request sent in two calls, each numbering its rows from 0, gives + every entity its own related rows and keys. + + Assumptions: + - The request is gathered as two calls, user 5 then user 7, and both + calls number their instances and related rows from the same start. + - The NIM answers each call with positions in that call's tables. + """ + client, transport = client_answering(gradient_explanation) + + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + result = client.predict( + split((5,), (7,)), model=KumoRelational(), outputs=list(EXPLAIN) + ) + + assert len(transport.sent) == 2 + assert sorted(result.explanations) == [5, 7] + seven = result.explanations[7]['related'] + assert seven['users']['keys'] == [7] + assert seven['orders']['keys'] == [WIDE] + assert seven['events']['rows'] == [ + {'user_id': 7, 'kind': 'logout', 'seconds': 71} + ] + assert result.explanations[5]['related']['orders']['keys'] == [9003, 9004] + + +def test_a_split_request_keeps_row_index_keys_apart(gradient_explanation): + """Explanations kept under ``row_index`` move with their rows when a later + call's rows are numbered after the earlier ones. + + Assumptions: + - Each call's answer has no entity key, so it is keyed by ``row_index``, + and each call numbers its rows from 0. + - The client numbers the second call's rows after the first call's two. + """ + from sdm_client.adapters.tabular import TabularAdapter + from sdm_client.client import _numbered_across_calls + from sdm_client.result import PredictionResult + + def answer(payload: dict) -> PredictionResult: + return TabularAdapter().normalize( + { + 'id': 'r', + 'model': 'kumo-tabular', + 'predictions': [ + { + 'row_index': index, + 'prediction': 1, + 'explanation': gradient_explanation( + outputs=CLASSES, + row=index, + columns=['age'], + gradients=[[-row[0]], [row[0]]], + ), + } + for index, row in enumerate( + payload['predict']['instance_table']['rows'] + ) + ], + 'metadata': {}, + }, + ('prediction', 'explanation'), + ) + + calls = [ + {'predict': {'instance_table': {'rows': [[10.0], [11.0]]}}}, + {'predict': {'instance_table': {'rows': [[12.0], [13.0]]}}}, + ] + + merged = PredictionResult.merge(_numbered_across_calls(calls, answer)) + + assert sorted(merged.explanations) == [0, 1, 2, 3] + assert merged.explanations[2]['instance']['gradients'] == [[-12.0], [12.0]] + assert sorted(merged.explanations) == list(merged.to_pandas()['row_index']) + + +# --- a forecast --------------------------------------------------------------- + + +def forecast(gradient_explanation, steps: int = 2): + """A forecast of users 5 and 7 over ``steps`` steps, through the client's + decomposed route, each step answered by the adapter as the NIM would.""" + from sdm_client.decompose import Decomposition + + rewritten = Decomposition( + 'forecast', tuple((f'q{step}', step) for step in range(1, steps + 1)) + ) + request = SimpleNamespace( + _decomposition=rewritten, parts=tuple(object() for _ in range(steps)) + ) + adapter = RelationalAdapter() + answers = [] + for _ in range(steps): + payload = gathered(5, 7, task='regression') + answer = nim_answer(payload, gradient_explanation) + for record in answer['predictions']: + record['prediction'] = 1.0 + del record['probabilities'] + answers.append(adapter.normalize(answer, EXPLAIN, payload)) + client = SDMClient.__new__(SDMClient) + handed = iter(answers) + client.predict = lambda part, model, outputs: next(handed) + return SDMClient._answer_in_parts( + client, request, 'kumo-relational', ['prediction', 'explanation'] + ) + + +def test_a_forecast_is_explained_under_id_and_step(gradient_explanation): + """Each point of a forecast is explained under ``(id, forecast_step)``, + and nothing is gathered under ``by_forecast_step``. + + Assumptions: + - Two steps, each answered with an explanation for users 5 and 7. + """ + result = forecast(gradient_explanation) + + assert sorted(result.explanations) == [(5, 1), (5, 2), (7, 1), (7, 2)] + assert 'by_forecast_step' not in result.explanations + frame = result.to_pandas() + assert sorted(result.explanations) == sorted( + zip(frame['id'], frame['forecast_step'], strict=True) + ) + + +def test_a_forecasts_flat_table_names_the_step(gradient_explanation): + """``to_pandas()`` of a forecast's explanations starts with ``id`` and + ``forecast_step``. + + Assumptions: + - Two steps, each explained. + """ + flat = forecast(gradient_explanation).explanations.to_pandas() + + assert list(flat.columns)[:3] == ['id', 'forecast_step', 'table'] + assert sorted(set(flat['forecast_step'])) == [1, 2] + + +# --- the flat table --------------------------------------------------------- + + +def test_the_flat_table_has_the_documented_columns(explained): + """``to_pandas()`` names the row, then where each gradient is. + + Assumptions: + - The entities are keyed by id. + """ + assert list(explained.explanations.to_pandas().columns) == [ + 'id', + 'table', + 'hop', + 'key', + 'row', + 'output', + 'column', + 'gradient', + ] + + +def test_the_flat_table_has_a_line_per_row_table_output_and_column(explained): + """One line per (entity, table, related row, output, column). + + Assumptions: + - Each block has one column and two outputs. User 5 has 1 instance row + and 1 + 2 + 2 + 2 related rows; user 7 has 1 and 1 + 1 + 1 + 1. + """ + flat = explained.explanations.to_pandas() + + assert len(flat) == 2 * (1 + 7) + 2 * (1 + 4) + assert len(flat[(flat['id'] == 5) & (flat['table'] == 'orders')]) == 4 + + +def test_the_flat_table_counts_hops_from_the_entity(explained): + """``hop`` is 0 for the instance and the entity's own row, 1 for a row + linked to it, and 2 for a row linked through one of those. + + Assumptions: + - ``users`` is reached from the instance, ``orders`` and ``events`` link + to ``users``, and ``products`` is linked from ``orders``. + """ + flat = explained.explanations.to_pandas() + linked = flat[(flat['id'] == 5) & (flat['key'] != 88)] + + hops = linked.groupby('table')['hop'].unique() + + assert {name: list(found) for name, found in hops.items()} == { + 'instance': [0], + 'users': [0], + 'orders': [1], + 'events': [1], + 'products': [2], + } + + +def test_the_flat_table_leaves_the_hop_of_an_unlinked_row_empty(explained): + """A row no relationship sent reaches from the entity has no hop. + + Assumptions: + - Product 88 is sampled for user 5, and no order of theirs names it. + """ + flat = explained.explanations.to_pandas() + + unlinked = flat[(flat['table'] == 'products') & (flat['key'] == 88)] + + assert len(unlinked) == 2 + assert unlinked['hop'].isna().all() + + +def test_the_flat_table_names_rows_by_key_and_position(explained): + """``key`` is a related row's key, empty for a keyless table, and the id + for the instance block; ``row`` is the position in ``rows``. + + Assumptions: + - User 5 has two orders and two keyless events. + """ + flat = explained.explanations.to_pandas() + five = flat[flat['id'] == 5] + + orders = five[five['table'] == 'orders'] + assert sorted(set(zip(orders['row'], orders['key'], strict=True))) == [ + (0, 9003), + (1, 9004), + ] + assert five[five['table'] == 'events']['key'].isna().all() + assert set(five[five['table'] == 'instance']['key']) == {5} + gradient = orders[ + (orders['row'] == 1) & (orders['output'] == 'probability_yes') + ]['gradient'] + assert gradient.tolist() == [13.0] + + +def test_the_flat_table_names_each_output_as_the_frame_does(explained): + """``output`` is the name of the column of ``result.to_pandas()`` holding + that score. + + Assumptions: + - The NIM explains both class probabilities and returns them. + """ + named = set(explained.explanations.to_pandas()['output']) + + assert named == {'probability_no', 'probability_yes'} + assert named <= set(explained.to_pandas().columns) diff --git a/packages/structured-data-models-client/tests/test_tabular_explanations.py b/packages/structured-data-models-client/tests/test_tabular_explanations.py index af3b1a82a..b8d0fcd08 100644 --- a/packages/structured-data-models-client/tests/test_tabular_explanations.py +++ b/packages/structured-data-models-client/tests/test_tabular_explanations.py @@ -10,6 +10,7 @@ from __future__ import annotations +import pandas as pd import pytest from sdm_client.adapters.tabular import TabularAdapter @@ -17,6 +18,10 @@ from sdm_client.models import KUMO_TABULAR EXPLAIN = ('prediction', 'explanation') +CLASSES = [ + {'kind': 'probability', 'class': '0'}, + {'kind': 'probability', 'class': '1'}, +] def response(**record) -> dict: @@ -81,24 +86,18 @@ def test_session_rows_sent(tabular_request): assert envelope['output']['fields'] == list(EXPLAIN) -def test_result_carries_explanation(): +def test_result_carries_explanation(gradient_explanation): """The row's explanation reaches the result under its row_index. Assumptions: - The endpoint returns one record, with no id, and its explanation. """ - explanation = { - 'format': 'sdm_gradient_v1', - 'details': { - 'outputs': [{'kind': 'prediction', 'statistic': 'mean'}], - 'instance_table': { - 'row_indices': [0], - 'columns': ['x'], - 'gradients': [[[0.5]]], - }, - 'related_tables': {}, - }, - } + explanation = gradient_explanation( + outputs=[{'kind': 'prediction', 'statistic': 'mean'}], + row=0, + columns=['x'], + gradients=[[0.5]], + ) result = TabularAdapter().normalize( response(explanation=explanation), EXPLAIN ) @@ -119,3 +118,246 @@ def test_result_without_explanation(): """ result = TabularAdapter().normalize(response(), ('prediction',)) assert result.explanations == {} + + +def answered(request, gradient_explanation, *, keyed=True, outputs=None): + """Send ``request`` through the adapter and read back a mocked answer. + + Each scored row ``i`` comes back as a NIM answers it: under ``'[]'`` + when the request has a key, with probabilities, and with its own + explanation, whose gradients are ``-(i + 1)`` and ``i + 1`` per class. + """ + adapter = TabularAdapter() + (envelope,) = adapter.prediction_plan(request, EXPLAIN) + table = envelope['predict']['instance_table'] + records = [] + for index, row in enumerate(table['rows']): + record = { + 'row_index': index, + 'prediction': 1, + 'probabilities': {'0': 0.25, '1': 0.75}, + 'explanation': gradient_explanation( + outputs=outputs or CLASSES, + row=index, + columns=['age'], + gradients=[[-(index + 1.0)], [index + 1.0]], + ), + } + if keyed: + record['id'] = f'[{row[table["columns"].index("row_id")]}]' + records.append(record) + return adapter.normalize( + { + 'id': 'r', + 'model': 'kumo-tabular', + 'predictions': records, + 'metadata': {}, + }, + EXPLAIN, + envelope, + ) + + +def test_every_row_is_explained_under_its_id( + tabular_request, gradient_explanation +): + """Each scored row's explanation is under that row's id, as the frame + names it. + + Assumptions: + - The request's rows carry the key ``row_id`` (90 and 91). + - The NIM answers each row under ``'[]'`` with its own explanation. + """ + result = answered(tabular_request(rows=2), gradient_explanation) + + assert sorted(result.explanations) == [90, 91] + assert sorted(result.explanations) == sorted(result.to_pandas()['id']) + + +def test_each_row_keeps_its_own_gradients( + tabular_request, gradient_explanation +): + """A row's instance gradients are its own, as ``[output][column]``. + + Assumptions: + - The NIM gives row ``i`` the gradients ``-(i + 1)`` and ``i + 1``. + """ + result = answered(tabular_request(rows=2), gradient_explanation) + + assert result.explanations[90]['instance'] == { + 'columns': ['age'], + 'gradients': [[-1.0], [1.0]], + } + assert result.explanations[91]['instance']['gradients'] == [[-2.0], [2.0]] + + +def test_rows_without_a_key_are_explained_under_their_row_index( + tabular_request, gradient_explanation +): + """With no key column, each explanation is under the row's row_index. + + Assumptions: + - The request has no ``row_id`` column, so the NIM sends no id. + """ + result = answered( + tabular_request(rows=3, keyed=False), gradient_explanation, keyed=False + ) + + assert sorted(result.explanations) == [0, 1, 2] + assert result.explanations[2]['instance']['gradients'] == [[-3.0], [3.0]] + + +def test_a_tabular_entry_has_no_related_tables( + tabular_request, gradient_explanation +): + """kumo-tabular has one table, so an entry carries no ``related`` key. + + Assumptions: + - The NIM sends an empty ``related_tables``, as it always does here. + """ + result = answered(tabular_request(rows=2), gradient_explanation) + + for entry in result.explanations.values(): + assert set(entry) == {'outputs', 'instance'} + + +def test_outputs_and_columns_are_passed_through( + tabular_request, gradient_explanation +): + """``outputs`` and ``columns`` are kept as the NIM returned them. + + Assumptions: + - The NIM names a quantile output with a float level and an extra key, + which the client does not read or rewrite. + """ + outputs = [ + {'kind': 'prediction', 'statistic': 'median'}, + {'kind': 'quantile', 'level': 0.1, 'note': 'kept'}, + ] + result = answered( + tabular_request(rows=1), gradient_explanation, outputs=outputs + ) + + assert result.explanations[90]['outputs'] == outputs + assert result.explanations[90]['instance']['columns'] == ['age'] + + +def test_the_flat_table_has_a_line_per_row_output_and_column( + tabular_request, gradient_explanation +): + """``to_pandas()`` gives one line per (row, output, column), keyed as the + frame is, each on the row's own block at hop 0. + + Assumptions: + - Two rows, two class outputs and one column: four lines. + """ + result = answered(tabular_request(rows=2), gradient_explanation) + + flat = result.explanations.to_pandas() + + assert list(flat.columns) == [ + 'id', + 'table', + 'hop', + 'key', + 'row', + 'output', + 'column', + 'gradient', + ] + assert len(flat) == 4 + assert set(flat['table']) == {'instance'} + assert set(flat['hop']) == {0} + assert list(flat['key']) == list(flat['id']) + assert flat.loc[flat['id'] == 91, 'gradient'].tolist() == [-2.0, 2.0] + + +def test_the_flat_table_names_a_keyless_row_by_its_row_index( + tabular_request, gradient_explanation +): + """With no key, ``to_pandas()`` names a row by ``row_index``, and its + ``key`` is empty. + + Assumptions: + - The request has no key column. + """ + result = answered( + tabular_request(rows=2, keyed=False), gradient_explanation, keyed=False + ) + + flat = result.explanations.to_pandas() + + assert list(flat.columns)[0] == 'row_index' + assert sorted(set(flat['row_index'])) == [0, 1] + assert flat['key'].isna().all() + + +@pytest.mark.parametrize( + ('task', 'outputs', 'returned'), + [ + ( + 'classification', + CLASSES, + {'probabilities': {'0': 0.25, '1': 0.75}}, + ), + ( + 'regression', + [ + {'kind': 'prediction', 'statistic': 'mean'}, + {'kind': 'quantile', 'level': 0.1}, + {'kind': 'quantile', 'level': 0.9}, + ], + {'quantiles': {'0.1': 1.0, '0.9': 3.0}}, + ), + ], + ids=['probabilities', 'quantiles'], +) +def test_the_flat_table_names_each_output_as_the_frame_does( + tabular_request, gradient_explanation, task, outputs, returned +): + """Each explained score is named by the column of ``result.to_pandas()`` + that holds it, so a gradient can be read beside its score. + + Assumptions: + - The NIM returns the scores it explains: every class probability, or the + prediction and each asked quantile keyed by its level as text. + """ + adapter = TabularAdapter() + (envelope,) = adapter.prediction_plan(tabular_request(task=task), EXPLAIN) + result = adapter.normalize( + response( + id='[90]', + explanation=gradient_explanation( + outputs=outputs, + row=0, + columns=['age'], + gradients=[[0.1]] * len(outputs), + ), + **returned, + ), + EXPLAIN, + envelope, + ) + + named = result.explanations.to_pandas()['output'] + + assert len(set(named)) == len(outputs) + assert set(named) <= set(result.to_pandas().columns) + + +def test_the_flat_table_of_no_explanation_is_empty_with_its_columns( + tabular_request, +): + """With nothing explained, ``to_pandas()`` is an empty frame that still + has its columns. + + Assumptions: + - The response carries no explanation. + """ + result = TabularAdapter().normalize(response(), ('prediction',)) + + flat = result.explanations.to_pandas() + + assert isinstance(flat, pd.DataFrame) + assert flat.empty + assert 'gradient' in flat.columns From b2a30cd3e870a04406564a203482655fd23beead Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Thu, 29 Oct 2026 13:25:39 -0700 Subject: [PATCH 09/25] Hold the in-process refusal of explanations where only it applies The parity case expected a kumo-relational NIM to refuse an explanation with UNSUPPORTED_OUTPUT, which a NIM that explains every row no longer does. It is removed; the in-process suite now checks that a model run in this process refuses 'explanation' for both models. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Jana Gagacheva --- .../test_a_model_runs_in_this_process.py | 35 +++++++++++++++++++ .../tests/test_every_backend_answers_alike.py | 9 ----- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/packages/structured-data-models-client/tests/test_a_model_runs_in_this_process.py b/packages/structured-data-models-client/tests/test_a_model_runs_in_this_process.py index 9f0e8b299..53f5af31a 100644 --- a/packages/structured-data-models-client/tests/test_a_model_runs_in_this_process.py +++ b/packages/structured-data-models-client/tests/test_a_model_runs_in_this_process.py @@ -431,6 +431,41 @@ def test_an_output_option_is_refused_before_the_model_runs( assert ran == [] +@pytest.mark.parametrize('model', ['tabular', 'relational']) +def test_an_explanation_is_refused_in_this_process(request, model): + """A model in this process produces no explanation, so asking for one is + refused as an output it does not give, rather than answered without it. + + Assumptions: + - The client sends ``'explanation'``, since both models declare it; only + a NIM serves it. + - Small random-weight models stand in for the published ones. + """ + served = request.getfixturevalue(model) + if model == 'tabular': + frame = people()[['person_id', 'age', 'visits', 'churn']] + asked = Request( + context=frame.iloc[:45], + predict=frame.iloc[45:].drop(columns=['churn']), + target='churn', + task='classification', + ) + handle = KumoTabular() + else: + asked = PredictiveQuery(BINARY, graph()).materialize(indices=[0, 1]) + handle = KumoRelational() + + with SDMClient.in_process(served) as client: + error = refused( + lambda: client.predict( + asked, model=handle, outputs=['prediction', 'explanation'] + ) + ) + + assert error.code == 'UNSUPPORTED_OUTPUT' + assert 'explanation' in str(error) + + def test_releasing_a_session_does_not_wait_for_a_model_to_finish( tabular, monkeypatch ): diff --git a/packages/structured-data-models-client/tests/test_every_backend_answers_alike.py b/packages/structured-data-models-client/tests/test_every_backend_answers_alike.py index 5aa0212de..a3b02b363 100644 --- a/packages/structured-data-models-client/tests/test_every_backend_answers_alike.py +++ b/packages/structured-data-models-client/tests/test_every_backend_answers_alike.py @@ -289,15 +289,6 @@ def _text_schema() -> Schema: ), 'UNSUPPORTED_OUTPUT', ), - ( - 'kumo-relational', - lambda client: client.predict( - PredictiveQuery(COUNT, graph()).materialize(indices=[0]), - model=KumoRelational(), - outputs=['prediction', 'explanation'], - ), - 'UNSUPPORTED_OUTPUT', - ), ( 'kumo-relational', lambda client: client.predict( From 36de54755ab9efbf454addc7703822240a7b8d2e Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Thu, 29 Oct 2026 13:25:39 -0700 Subject: [PATCH 10/25] Name the context-size tests for what they still check The explanation caps they were named after are gone; the tests still check that a gathered payload is sent in one call. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Jana Gagacheva --- .../tests/test_limits_and_sessions.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/structured-data-models-client/tests/test_limits_and_sessions.py b/packages/structured-data-models-client/tests/test_limits_and_sessions.py index 005eaa844..6488474d0 100644 --- a/packages/structured-data-models-client/tests/test_limits_and_sessions.py +++ b/packages/structured-data-models-client/tests/test_limits_and_sessions.py @@ -86,7 +86,12 @@ def test_explains_large_context(): assert plans[0]['output']['fields'] == ['prediction', 'explanation'] -def test_an_explanation_within_both_limits_is_allowed(): +def test_an_explanation_over_a_small_context_is_sent(): + """An explanation over a 500-row context is sent in one call. + + Assumptions: + - The payload scores one entity. + """ request = _Gathered(payload(predict_rows=1, context_rows=500)) plans = list( RelationalAdapter().prediction_plan( @@ -97,8 +102,13 @@ def test_an_explanation_within_both_limits_is_allowed(): assert plans[0]['output']['fields'] == ['prediction', 'explanation'] -def test_a_prediction_without_an_explanation_is_not_held_to_those_limits(): - """Both caps are explanation-only; an ordinary batch is unaffected.""" +def test_a_prediction_of_many_rows_over_a_large_context_is_one_call(): + """An ordinary batch of 500 rows over a 5,000-row context is sent as it + was gathered, in one call. + + Assumptions: + - The request was gathered as one payload, within every per-call cap. + """ request = _Gathered(payload(predict_rows=500, context_rows=5_000)) plans = list(RelationalAdapter().prediction_plan(request, ('prediction',))) assert len(plans) == 1 From bff621aba3a25ece3ec71ff0bac8f75093e5da20 Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Thu, 29 Oct 2026 13:29:30 -0700 Subject: [PATCH 11/25] Document relational explanations without the engine's question row A kumo-relational entry is only outputs and related; the entity's own table sits under related at hop 0, and the flat table has no instance line for it. The relational probe checks for that. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jana Gagacheva --- CHANGELOG.md | 9 +-- docs/about/architecture.md | 14 ++--- docs/reference/prediction-output.md | 56 ++++++++----------- qa/probe_relational.py | 22 ++++---- skills/relational/SKILL.md | 12 ++-- .../references/beyond-one-answer.md | 13 ++--- 6 files changed, 56 insertions(+), 70 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4b7fd6b3..bb08b2450 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,10 +32,11 @@ these packages. All five distributions share one version and one entry. - `kumo-tabular` accepts `explanation` in `outputs`. - `result.explanations` maps each explained row's key to its explanation: the row's `id`, `(id, forecast_step)` on a forecast, or `row_index` when the - request has no entity id. Each entry is a plain dict of `outputs`, the - entity's own row under `instance`, and, on `kumo-relational`, each related - table under `related`, whose rows carry their primary key when the table - has one and always their values as sent. It is kept out of + request has no entity id. Each entry is a plain dict of `outputs` and, on + `kumo-tabular`, the row's own columns under `instance`, or, on + `kumo-relational`, every table, the entity's own included, under `related`, + whose rows carry their primary key when the table has one and always their + values as sent. It is kept out of `result.to_pandas()`; `result.explanations.to_pandas()` flattens it into one line per row, output and column, with each row's `hop` from the entity. diff --git a/docs/about/architecture.md b/docs/about/architecture.md index 7376c42ab..f17f558a4 100644 --- a/docs/about/architecture.md +++ b/docs/about/architecture.md @@ -611,14 +611,12 @@ it on the result. `result.explanations.to_pandas()` flattens them into a table of their own when one is wanted, with each related row's `hop` from the entity worked out from the relationships and foreign-key values the client sent. -**The relational entity is two tables.** The engine sends the per-prediction -row, the entity key, anchor time and its own bookkeeping, as the instance -table, and the entity's own feature row as a related table sampled like any -other. So a `kumo-relational` entry's `instance` holds the bookkeeping row's -gradients, and the entity's features are under `related`, by their table's -name. The flat table shows the entity on an `instance` line and on a line of -its own table, both hop 0. A `kumo-tabular` entry has one table, the -`instance`. +**The engine's question row is dropped.** The engine sends a question row +of its own, the entity key and anchor time with no caller columns, beside the +entity's own table, and the endpoint explains both. The client drops that +row's block when it reads the reply, because nothing in it is the caller's, so +a `kumo-relational` entry is only `related`, with the entity's own table at +hop 0. A forecast is explained a step at a time, under `(id, forecast_step)`, rather than aggregated across steps: each step is its own row in `to_pandas()`, and diff --git a/docs/reference/prediction-output.md b/docs/reference/prediction-output.md index cf824df6b..502f1904e 100644 --- a/docs/reference/prediction-output.md +++ b/docs/reference/prediction-output.md @@ -149,16 +149,17 @@ both models: ### One entry -Each entry is a plain nested dict, the same shape for both models: +Each entry is a plain nested dict: ```python { 'outputs': [...], # the explained scores, in order - 'instance': { + 'instance': { # kumo-tabular only 'columns': [...], # model's preprocessed numerical columns 'gradients': [[...], ...], # [output][column] }, - 'related': { # absent for kumo-tabular + 'related': { # kumo-relational only: every table, + # the entity's own included '
': { 'keys': [...], # the table's primary key per row; absent if the table has none 'rows': [{...}, ...], # each row's values as sent to the model @@ -193,12 +194,8 @@ table without numerical features has an empty `columns`. | kumo-relational | the predicted entity's own table, plus every related table in the subgraph the client gathered for that entity (all tables reached within the hop limit, up to 6) | A `kumo-tabular` entry has no `related` key: its one table is the `instance`. - -`kumo-relational` sends the entity as two tables. `instance` is the -per-prediction row the engine builds: the entity key, the anchor time and the -engine's own bookkeeping, with few or no numerical columns. The entity's own -feature row is a related table like any other, under its table's name, such -as `related['users']`, and that is where its features' gradients are. +A `kumo-relational` entry has no `instance` key: the entity's own table is one +of the tables under `related`, at hop 0. Only the rows being predicted and their related rows get gradients. The context rows, the labelled examples the model compares against, get none. @@ -228,18 +225,14 @@ result.explanations['cust_17'] # 'gradients': [[0.01, -0.03], [-0.01, 0.03]]}} ``` -`kumo-relational`, user 5 with one order and two login events. The user's own -features are under `related['users']`; `instance` holds the engine's -per-prediction row. `orders` has a primary key (`order_id`), so its rows carry -`keys`. `events` has none, so its rows are identified by their values alone. -All are part of the same entry: +`kumo-relational`, user 5 with one order and two login events. `orders` has a +primary key (`order_id`), so its rows carry `keys`. `events` has none, so its +rows are identified by their values alone. Both are part of the same entry: ```python result.explanations['user_5'] # {'outputs': [{'kind': 'probability', 'class': 'no'}, # {'kind': 'probability', 'class': 'yes'}], -# 'instance': {'columns': ['anchor_time_dayofweek'], # engine bookkeeping, illustrative -# 'gradients': [[0.001], [-0.001]]}, # 'related': { # 'users': {'keys': ['user_5'], # 'rows': [{'user_id': 'user_5', 'tenure_days': 410}], @@ -262,23 +255,22 @@ result.explanations['user_5'] across all entities: ``` -id table hop key row output column gradient -user_5 instance 0 user_5 0 probability_yes anchor_time_dayofweek -0.001 -user_5 users 0 user_5 0 probability_yes tenure_days 0.05 -user_5 orders 1 order_9003 0 probability_yes amount 0.02 -user_5 products 2 prod_77 0 probability_yes price 0.004 -user_5 events 1 None 1 probability_yes kind_login -0.02 +id table hop key row output column gradient +user_5 users 0 user_5 0 probability_yes tenure_days 0.05 +user_5 orders 1 order_9003 0 probability_yes amount 0.02 +user_5 products 2 prod_77 0 probability_yes price 0.004 +user_5 events 1 None 1 probability_yes kind_login -0.02 ``` -- `table` is `instance` for the entry's `instance` block and the table's name - for a related one. On `kumo-relational` the entity shows on both an - `instance` line and a line of its own table (`users` here), both hop 0. -- `hop` is how many links the row is from the entity, 0 for the entity - itself. The client works it out from the relationships and foreign-key - values it sent. It is `None` for a related row no relationship reaches. -- `key` is the row's primary key, or `None` for a keyless table. On an - `instance` line it is the row's `id`, or `None` when the request has no - entity key. `row` is the row's position in that entity's - `related[table]['rows']`, and 0 on an `instance` line. +- `table` is the table's name. On `kumo-relational` the entity's own table + (`users` here) is hop 0. On `kumo-tabular` the one table is shown as + `instance`, at hop 0. +- `hop` is how many links the row is from the entity, 0 for the entity's own + table. The client works it out from the relationships and foreign-key + values it sent. It is `None` for a row no relationship reaches. +- `key` is the row's primary key, or `None` for a keyless table. On a + `kumo-tabular` `instance` line it is the row's `id`, or `None` when the + request has no entity key. `row` is the row's position in that entity's + `related[table]['rows']`, and 0 on a `kumo-tabular` `instance` line. - `output` names the score by the column of `result.to_pandas()` that holds it: `probability_yes`, `prediction`, `quantile_0.1`. diff --git a/qa/probe_relational.py b/qa/probe_relational.py index b9c9b58a0..eb1a969bd 100644 --- a/qa/probe_relational.py +++ b/qa/probe_relational.py @@ -456,22 +456,22 @@ def rows(request): ) elif not all(entry.get('related') for entry in explained.values()): fail(e, 'an entry carries no related tables') + elif any('instance' in entry for entry in explained.values()): + fail(e, 'an entry carries the engine question row as instance') + elif not all( + 'Orders' in entry['related'] for entry in explained.values() + ): + fail(e, "an entry lacks the entity's own table, Orders") elif any( - len(entry['instance']['gradients']) != len(entry['outputs']) + len(block['gradients']) != len(entry['outputs']) or any( - len(block['gradients']) != len(entry['outputs']) - or any( - len(per_output) != len(block['rows']) - for per_output in block['gradients'] - ) - for block in entry['related'].values() + len(per_output) != len(block['rows']) + for per_output in block['gradients'] ) for entry in explained.values() + for block in entry['related'].values() ): - fail( - e, - 'gradients are not [output][column] and [output][row][column]', - ) + fail(e, 'gradients are not [output][row][column]') else: flat = explained.to_pandas() if 'explanation' in frame.columns: diff --git a/skills/relational/SKILL.md b/skills/relational/SKILL.md index 28d2526b9..41439c02c 100644 --- a/skills/relational/SKILL.md +++ b/skills/relational/SKILL.md @@ -191,20 +191,18 @@ of its own, and they are in An explanation is kept out of `to_pandas()`, on `result.explanations`, a mapping from each row's key to its explanation: the entity's `id`, or -`(id, forecast_step)` on a forecast. One entry covers the engine's -per-prediction row, the entity's own feature row and every related row -gathered for it. The entity's features are under its table's name in -`related`, not in `instance`: +`(id, forecast_step)` on a forecast. One entry covers the entity's own row +and every related row gathered for it, each under its table's name in +`related`: ```python entry = result.explanations['user_5'] # the entity whose id is 'user_5' -entry['instance']['gradients'] # [output][column], the per-prediction row: key, anchor time, bookkeeping -entry['related']['users'] # the entity's own features, one row +entry['related']['users'] # the entity's own row, hop 0 orders = entry['related']['orders'] # one block per related table orders['keys'] # each row's primary key; absent if the table has none orders['rows'] # each row's values as sent orders['gradients'] # [output][row][column] -result.explanations.to_pandas() # one line per (row, output, column), with its hop; the entity is hop 0 on both its instance and users lines +result.explanations.to_pandas() # one line per (row, output, column), with its hop; the entity's users line is hop 0 ``` ## Asking the same context more than once diff --git a/skills/relational/references/beyond-one-answer.md b/skills/relational/references/beyond-one-answer.md index 38b340731..dc130cb2f 100644 --- a/skills/relational/references/beyond-one-answer.md +++ b/skills/relational/references/beyond-one-answer.md @@ -119,13 +119,10 @@ explanation, the same key that identifies the row in `to_pandas()`: the entity's `id`, or `(id, forecast_step)` on a forecast, since a forecast is keyed by the pair. It is kept out of `to_pandas()` itself. -An entry holds `outputs`, the explained scores in order; `instance`, the -engine's per-prediction row (the entity key, anchor time and its own -bookkeeping, with few or no numerical columns), with `columns` and `gradients` -as `[output][column]`; and `related`, one block per related table the client -gathered for that entity. The entity's own feature row is one of those blocks, -under its table's name, such as `related['users']`, so its features' gradients -are there and not in `instance`. +An entry holds `outputs`, the explained scores in order, and `related`, one +block per table the client gathered for that entity. The entity's own row is +one of those blocks, under its table's name, such as `related['users']`, at +hop 0. A block carries `keys`, each row's primary key, when the table has one, and always `rows`, each row's values as sent, with `columns` and `gradients` as `[output][row][column]`, aligned with those rows. A keyless table's rows are @@ -137,7 +134,7 @@ features gives an empty list. The context rows get no gradients. A row reached through several paths appears once, with its total effect through all of them. `result.explanations.to_pandas()` flattens every entry into one line per (row, output, column), with `hop`, how many links the row is from the entity. The -entity shows on an `instance` line and on a line of its own table, both hop 0. +entity's own table is hop 0. A ranking refuses an explanation locally, and the shipping endpoint refuses it everywhere else, so treat `result.explanations` as possibly empty. A deployment From ca99c782f6e8a304af2defe9f29c2b9a50681a6f Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Thu, 29 Oct 2026 13:31:31 -0700 Subject: [PATCH 12/25] Drop the engine's question row from a relational explanation The relational engine sends each entity as a question row in instance_table and the entity's own row in related_tables. The NIM explains both, but the question row carries no caller columns, so a kumo-relational entry now has only 'outputs' and 'related', where the entity's own table sits at hop 0. kumo-tabular entries keep 'instance' and have no 'related'. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jana Gagacheva --- .../src/sdm_client/result.py | 49 ++++++++++--------- .../src/sdm_client/wire/explanations.py | 21 ++++---- .../tests/test_relational_explanations.py | 35 ++++++------- 3 files changed, 52 insertions(+), 53 deletions(-) diff --git a/packages/structured-data-models-client/src/sdm_client/result.py b/packages/structured-data-models-client/src/sdm_client/result.py index 7c6937239..cf0d21780 100644 --- a/packages/structured-data-models-client/src/sdm_client/result.py +++ b/packages/structured-data-models-client/src/sdm_client/result.py @@ -82,15 +82,17 @@ class Explanations(dict[Any, dict[str, Any]]): dict:: {'outputs': [...], # the explained scores - 'instance': {'columns': [...], + 'instance': {'columns': [...], # kumo-tabular only 'gradients': [...]}, # [output][column] 'related': {table: {'keys': [...], # absent for a keyless table 'rows': [{...}], # each row's values as sent 'columns': [...], 'gradients': [...]}}} # [output][row][column] - ``related`` is there for ``kumo-relational`` only. ``columns`` are the - model's preprocessed numerical columns, as the endpoint names them. + ``instance`` is there for ``kumo-tabular`` only, and ``related`` for + ``kumo-relational`` only, where it holds every table, the entity's own + among them. ``columns`` are the model's preprocessed numerical columns, as + the endpoint names them. Empty when no explanation was asked for, or none came back. """ @@ -114,8 +116,9 @@ def to_pandas(self) -> pd.DataFrame: r"""Every gradient, one line each, across all explained rows. One line per (row, table, related row, output, column). ``table`` is - ``'instance'`` for the row's own block and the table's name for a - related one. ``hop`` is how many links the row is from the entity: 0 + ``'instance'`` for a ``kumo-tabular`` row's own block and the table's + name for a related one, the ``kumo-relational`` entity's own table + among them. ``hop`` is how many links the row is from the entity: 0 for the entity itself, and ``None`` where the rows sent do not link it. ``key`` is the related row's primary key, ``None`` for a keyless table, and the row's own ``id`` for its instance block. ``row`` is the @@ -126,26 +129,28 @@ def to_pandas(self) -> pd.DataFrame: for key, entry in self.items(): named = key if len(self._keyed_by) > 1 else (key,) outputs = [_output_named(output) for output in entry['outputs']] - instance = entry['instance'] + # A kumo-relational entry has none: its own row is in `related`. + instance = entry.get('instance') own = named[0] if self._keyed_by[0] == 'id' else None - for output, by_column in zip( - outputs, instance['gradients'], strict=True - ): - for column, gradient in zip( - instance['columns'], by_column, strict=True + if instance is not None: + for output, by_column in zip( + outputs, instance['gradients'], strict=True ): - lines.append( - ( - *named, - 'instance', - 0, - own, - 0, - output, - column, - gradient, + for column, gradient in zip( + instance['columns'], by_column, strict=True + ): + lines.append( + ( + *named, + 'instance', + 0, + own, + 0, + output, + column, + gradient, + ) ) - ) hops = self._hops.get(key, {}) for table, block in entry.get('related', {}).items(): keys = block.get('keys') or [None] * len(block['rows']) diff --git a/packages/structured-data-models-client/src/sdm_client/wire/explanations.py b/packages/structured-data-models-client/src/sdm_client/wire/explanations.py index 54155cd9e..646b90a27 100644 --- a/packages/structured-data-models-client/src/sdm_client/wire/explanations.py +++ b/packages/structured-data-models-client/src/sdm_client/wire/explanations.py @@ -37,8 +37,11 @@ def explanations_of( ``items`` have their ``id`` restored already, so the key is the one the frame shows: the ``id``, or the ``row_index`` when the rows carry none. - ``related`` is given for a model that reads related tables; without it an - entry has no ``related`` at all, rather than an empty one. + ``related`` is given for a model that reads related tables. Then an entry + has ``related`` and no ``instance``: that model's ``instance_table`` is the + question row its engine sends, not the caller's, and the entity's own table + is among the related ones. Without it an entry has ``instance`` and no + ``related`` at all, rather than an empty one. """ explained = [item for item in items if item.explanation] if not explained: @@ -49,18 +52,17 @@ def explanations_of( for item in explained: key = item.id if by_id else item.row_index details = item.explanation.get('details') or {} - instance = details.get('instance_table') or {} - entry: dict[str, Any] = { - 'outputs': list(details.get('outputs') or []), - 'instance': { + entry: dict[str, Any] = {'outputs': list(details.get('outputs') or [])} + if related is None: + instance = details.get('instance_table') or {} + entry['instance'] = { 'columns': list(instance.get('columns') or []), # One row, the record's own, so the row axis says nothing. 'gradients': [ by_row[0] for by_row in instance.get('gradients') or [] ], - }, - } - if related is not None: + } + else: entry['related'], hops[key] = related( details.get('related_tables') or {} ) @@ -70,4 +72,3 @@ def explanations_of( keyed_by=('id',) if by_id else ('row_index',), hops=hops, ) - diff --git a/packages/structured-data-models-client/tests/test_relational_explanations.py b/packages/structured-data-models-client/tests/test_relational_explanations.py index 48c2b229a..f5ef6dc09 100644 --- a/packages/structured-data-models-client/tests/test_relational_explanations.py +++ b/packages/structured-data-models-client/tests/test_relational_explanations.py @@ -286,21 +286,15 @@ def test_every_entity_is_explained_under_its_id(explained): assert sorted(explained.explanations) == sorted(explained.to_pandas()['id']) -def test_an_entity_has_its_own_instance_gradients(explained): - """The instance block is the record's own, as ``[output][column]``. +def test_the_engines_question_row_is_not_an_entrys_block(explained): + """An entry has no ``instance``: that block explains the engine's question + row, not a row the caller sent, and the entity's own row is in ``related``. Assumptions: - - The NIM gives the record at row ``i`` the gradients ``-(i + 1)`` and - ``i + 1``. + - The NIM explains every record's ``instance_table`` as well. """ - assert explained.explanations[5]['instance'] == { - 'columns': ['anchor_dayofweek'], - 'gradients': [[-1.0], [1.0]], - } - assert explained.explanations[7]['instance']['gradients'] == [ - [-2.0], - [2.0], - ] + for entry in explained.explanations.values(): + assert set(entry) == {'outputs', 'related'} def test_an_entity_has_a_block_for_every_related_table(explained): @@ -398,7 +392,6 @@ def test_outputs_and_columns_are_passed_through(explained): entry = explained.explanations[5] assert entry['outputs'] == CLASSES - assert entry['instance']['columns'] == ['anchor_dayofweek'] assert entry['related']['events']['columns'] == ['seconds'] @@ -590,21 +583,21 @@ def test_the_flat_table_has_a_line_per_row_table_output_and_column(explained): """One line per (entity, table, related row, output, column). Assumptions: - - Each block has one column and two outputs. User 5 has 1 instance row - and 1 + 2 + 2 + 2 related rows; user 7 has 1 and 1 + 1 + 1 + 1. + - Each block has one column and two outputs. User 5 has 1 + 2 + 2 + 2 + related rows and user 7 has 1 + 1 + 1 + 1; the question row is dropped. """ flat = explained.explanations.to_pandas() - assert len(flat) == 2 * (1 + 7) + 2 * (1 + 4) + assert len(flat) == 2 * 7 + 2 * 4 assert len(flat[(flat['id'] == 5) & (flat['table'] == 'orders')]) == 4 def test_the_flat_table_counts_hops_from_the_entity(explained): - """``hop`` is 0 for the instance and the entity's own row, 1 for a row + """``hop`` is 0 for the entity's own row, 1 for a row linked to it, and 2 for a row linked through one of those. Assumptions: - - ``users`` is reached from the instance, ``orders`` and ``events`` link + - ``users`` holds the entity, ``orders`` and ``events`` link to ``users``, and ``products`` is linked from ``orders``. """ flat = explained.explanations.to_pandas() @@ -613,7 +606,6 @@ def test_the_flat_table_counts_hops_from_the_entity(explained): hops = linked.groupby('table')['hop'].unique() assert {name: list(found) for name, found in hops.items()} == { - 'instance': [0], 'users': [0], 'orders': [1], 'events': [1], @@ -637,7 +629,7 @@ def test_the_flat_table_leaves_the_hop_of_an_unlinked_row_empty(explained): def test_the_flat_table_names_rows_by_key_and_position(explained): """``key`` is a related row's key, empty for a keyless table, and the id - for the instance block; ``row`` is the position in ``rows``. + for the entity's own row; ``row`` is the position in ``rows``. Assumptions: - User 5 has two orders and two keyless events. @@ -651,7 +643,8 @@ def test_the_flat_table_names_rows_by_key_and_position(explained): (1, 9004), ] assert five[five['table'] == 'events']['key'].isna().all() - assert set(five[five['table'] == 'instance']['key']) == {5} + assert set(five[five['table'] == 'users']['key']) == {5} + assert 'instance' not in set(flat['table']) gradient = orders[ (orders['row'] == 1) & (orders['output'] == 'probability_yes') ]['gradient'] From 9de1214f8d67dd528344093f8474dfa18d2a5ba4 Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Thu, 29 Oct 2026 13:36:17 -0700 Subject: [PATCH 13/25] Test multi-row explanations against a live NIM kumo-tabular: several rows in one call are each explained under their id, or their row_index without a key column, with only 'outputs' and 'instance', gradients [output][column], and a flat table of the row's own block at hop 0. kumo-relational: several entities, in one call and split across calls that each number from 0, are each explained under their id with only 'outputs' and 'related'. The entity's own table is among the related ones at hop 0, each keyed row carries its key and the caller's values, and gradients line up with the rows. The skills test that expected the relational endpoint to refuse an explanation now expects one per entity in that shape. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jana Gagacheva --- .../tests/test_live_nim.py | 103 ++++++++++++++ .../tests/test_live_relational.py | 131 ++++++++++++++++++ tests/test_skills.py | 43 ++++-- 3 files changed, 267 insertions(+), 10 deletions(-) diff --git a/packages/structured-data-models-client/tests/test_live_nim.py b/packages/structured-data-models-client/tests/test_live_nim.py index c0cf0f616..6e34b8b1d 100644 --- a/packages/structured-data-models-client/tests/test_live_nim.py +++ b/packages/structured-data-models-client/tests/test_live_nim.py @@ -225,3 +225,106 @@ def test_a_tabular_question_runs_from_predictive_query_text(client): frame = client.predict(request, model=KumoTabular()).to_pandas() assert len(frame) == 3 assert 'prediction' in frame.columns + + +EXPLAIN = ['prediction', 'probabilities', 'explanation'] +FLAT_COLUMNS = ['table', 'hop', 'key', 'row', 'output', 'column', 'gradient'] + + +def explained_by_tabular(client, *, keyed: bool = True): + """Score ``unlabelled()`` with explanations, without ``row_id`` when + ``keyed`` is false.""" + context, predict = labelled(), unlabelled() + if not keyed: + context = context.drop(columns='row_id') + predict = predict.drop(columns='row_id') + request = Request( + context=context, + predict=predict, + target='churn', + task='classification', + ) + return client.predict(request, model=KumoTabular(), outputs=EXPLAIN) + + +def assert_instance_only(entry): + """A kumo-tabular entry: its outputs and its own row's gradients, + ``[output][column]``, and no related tables.""" + assert set(entry) == {'outputs', 'instance'} + # Both class probabilities of a binary target. + assert len(entry['outputs']) == 2 + instance = entry['instance'] + assert set(instance) == {'columns', 'gradients'} + assert instance['columns'], 'no column was explained' + assert len(instance['gradients']) == len(entry['outputs']) + for by_column in instance['gradients']: + assert len(by_column) == len(instance['columns']) + + +@requires_tabular +def test_every_scored_row_is_explained_under_its_id(client): + """Each of several rows scored in one call gets its own explanation, under + the id the frame gives it, holding only its own row's gradients. + + Assumptions: + - ``row_id`` is the rows' key, so the frame names them 900, 901, 902. + - The NIM explains every scored row and both class probabilities. + """ + result = explained_by_tabular(client) + frame = result.to_pandas() + + assert sorted(result.explanations) == sorted(frame['id']) + assert sorted(result.explanations) == [900, 901, 902] + for entry in result.explanations.values(): + assert_instance_only(entry) + assert 'explanation' not in frame.columns + + +@requires_tabular +def test_the_flat_explanation_table_holds_each_rows_own_block(client): + """``to_pandas()`` gives one line per (row, output, column), all of the + row's own block at hop 0, named by the frame's id and output columns. + + Assumptions: + - ``row_id`` is the rows' key, so each line's ``key`` is the row's id. + """ + result = explained_by_tabular(client) + frame = result.to_pandas() + flat = result.explanations.to_pandas() + + assert list(flat.columns) == ['id', *FLAT_COLUMNS] + assert len(flat) == sum( + len(entry['outputs']) * len(entry['instance']['columns']) + for entry in result.explanations.values() + ) + assert set(flat['table']) == {'instance'} + assert (flat['hop'] == 0).all() + assert (flat['row'] == 0).all() + assert list(flat['key']) == list(flat['id']) + assert set(flat['id']) == {900, 901, 902} + assert set(flat['output']) <= set(frame.columns) + assert flat['gradient'].notna().all() + + +@requires_tabular +def test_rows_without_a_key_are_explained_under_their_row_index(client): + """With no key column, each row's explanation is under its row_index, in + the same shape. + + Assumptions: + - The rows carry no ``row_id``, so the NIM sends no id. + """ + result = explained_by_tabular(client, keyed=False) + frame = result.to_pandas() + + assert sorted(result.explanations) == list(frame['row_index']) + assert sorted(result.explanations) == [0, 1, 2] + for entry in result.explanations.values(): + assert_instance_only(entry) + assert 'explanation' not in frame.columns + + flat = result.explanations.to_pandas() + assert list(flat.columns) == ['row_index', *FLAT_COLUMNS] + assert set(flat['row_index']) == {0, 1, 2} + assert set(flat['table']) == {'instance'} + assert (flat['hop'] == 0).all() diff --git a/packages/structured-data-models-client/tests/test_live_relational.py b/packages/structured-data-models-client/tests/test_live_relational.py index 064ced086..11e5a7619 100644 --- a/packages/structured-data-models-client/tests/test_live_relational.py +++ b/packages/structured-data-models-client/tests/test_live_relational.py @@ -328,3 +328,134 @@ def test_run_mode_is_refused_as_an_option(query): with pytest.raises(StructuredDataError) as error: query.materialize(indices=[0], run_mode='best') assert 'run_mode' in str(error.value) + + +EXPLAIN = ['prediction', 'probabilities', 'explanation'] +BINARY_QUERY = 'PREDICT COUNT(orders.*, 0, 30, days) > 2 FOR EACH users.user_id' +#: Each table of ``tables()`` and its primary key. Both have one, so every +#: related block is expected to carry ``keys``; the graph has no keyless table. +KEYS = {'users': 'user_id', 'orders': 'order_id'} + + +@pytest.fixture(scope='module') +def binary_query(): + return PredictiveQuery(BINARY_QUERY, tables()) + + +def assert_explained_as_sent(result, asked): + """Every asked entity is explained once, under its id, by its own rows. + + An entry has ``outputs`` and ``related`` only, the entity's own table + among the related ones. Each block's ``keys`` are the primary keys of its + ``rows``, whose values are the caller's for that key and belong to that + entity, and its gradients are ``[output][row][column]``, aligned with + ``rows``. In ``to_pandas()`` the entity's own table is at hop 0 and the + others at 1 or more. + """ + graph = { + name: frame.set_index(KEYS[name]) for name, frame in tables().items() + } + frame = result.to_pandas() + assert 'explanation' not in frame.columns + ids = {int(key): key for key in result.explanations} + assert len(ids) == len(result.explanations) + assert sorted(ids) == sorted(asked) + assert sorted(ids) == sorted(int(value) for value in frame['id']) + + for entity, key in ids.items(): + entry = result.explanations[key] + assert set(entry) == {'outputs', 'related'} + outputs = len(entry['outputs']) + # Both class probabilities of a binary target. + assert outputs == 2 + related = entry['related'] + assert 'users' in related, 'the entity has no block of its own' + assert set(related) <= set(KEYS) + for name, block in related.items(): + assert 'keys' in block, f'{name} has a key and came back without' + assert len(block['keys']) == len(block['rows']) + assert len(block['gradients']) == outputs + for by_row in block['gradients']: + assert len(by_row) == len(block['rows']) + for by_column in by_row: + assert len(by_column) == len(block['columns']) + for row_key, row in zip(block['keys'], block['rows'], strict=True): + # The caller's columns only: no instance_id, no __node_id. + assert set(row) <= {KEYS[name], *graph[name].columns} + assert int(row[KEYS[name]]) == int(row_key) + assert int(row['user_id']) == entity + sent = graph[name].loc[int(row_key)] + if name == 'orders': + assert row['price'] == sent['price'] + else: + assert row['segment'] == sent['segment'] + assert [int(one) for one in related['users']['keys']] == [entity] + + flat = result.explanations.to_pandas() + assert list(flat.columns) == [ + 'id', + 'table', + 'hop', + 'key', + 'row', + 'output', + 'column', + 'gradient', + ] + assert 'instance' not in set(flat['table']) + assert set(int(one) for one in flat['id']) == set(asked) + own = flat[flat['table'] == 'users'] + assert len(own), "the entity's own table has no gradient" + assert (own['hop'] == 0).all() + assert [int(one) for one in own['key']] == [int(one) for one in own['id']] + others = flat[flat['table'] != 'users'] + assert others['hop'].notna().all() + assert (others['hop'] >= 1).all() + + +@requires_relational +def test_every_entity_is_explained_with_its_own_related_rows( + client, binary_query +): + """Several entities in one call each get an explanation of their own, + whose related rows are the ones sent for that entity. + + Assumptions: + - The NIM explains every predict row, both probabilities of a binary + target. + - ``users`` and ``orders`` both have a primary key; the graph has no + keyless table, so the keyless case is not exercised here. + """ + asked = [0, 1, 2, 3] + request = binary_query.materialize(indices=asked) + assert len(request.payloads) == 1 + + result = client.predict(request, model=KumoRelational(), outputs=EXPLAIN) + + assert_explained_as_sent(result, asked) + + +@requires_relational +def test_a_request_sent_in_several_calls_explains_every_entity( + client, binary_query +): + """A request over the per-call cap is sent in calls that each number + their rows from 0, and every entity still gets one entry, with its own + related rows and keys. + + Assumptions: + - ``rows_per_call=2`` stands in for the cap, as in the batching test + above, and splits eight entities into four calls. + """ + import warnings + + asked = list(range(8)) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + request = binary_query.materialize(indices=asked, rows_per_call=2) + result = client.predict( + request, model=KumoRelational(), outputs=EXPLAIN + ) + + assert len(request.payloads) == 4 + assert_explained_as_sent(result, asked) diff --git a/tests/test_skills.py b/tests/test_skills.py index b364e0296..db26b0669 100644 --- a/tests/test_skills.py +++ b/tests/test_skills.py @@ -2475,21 +2475,44 @@ def test_the_relational_endpoint_serves_the_matrix_the_skill_gives( reason='set SDM_RELATIONAL_URL to a kumo-relational NIM', ) @pytest.mark.parametrize('target', sorted(SERVED), ids=lambda v: v[:30]) -def test_the_relational_endpoint_refuses_an_explanation(target, schema): - """The gap the skill leads with: the client sends it, the endpoint will - not serve it. If a deployment starts serving one, this fails and the - skill is the thing to change.""" +def test_the_relational_endpoint_explains_every_entity(target, schema): + """What the skill shows of an explanation: one per entity asked about, + under its id, with the entity's own table among its related ones and no + block for the engine's question row. If a deployment stops serving one, + this fails and the skill is the thing to change.""" from sdm_client import KumoRelational, SDMClient - from sdm_client.errors import StructuredDataError request = resolve(f'PREDICT {target} FOR EACH users.user_id', schema) with SDMClient(url=RELATIONAL_URL, timeout=600) as client: - with pytest.raises(StructuredDataError, match='explanation'): - client.predict( - request, - model=KumoRelational(), - outputs=['prediction', 'explanation'], + result = client.predict( + request, + model=KumoRelational(), + outputs=['prediction', 'explanation'], + ) + # `resolve` asks about users 1 and 2. + asked = [1, 2] + assert sorted(int(one) for one in result.to_pandas()['id']) == asked + assert sorted(int(key) for key in result.explanations) == asked, ( + f'{target} explained {sorted(result.explanations)}, asked {asked}' + ) + for key, entry in result.explanations.items(): + assert set(entry) == {'outputs', 'related'}, ( + f'{target} gave {key} the blocks {sorted(entry)}' + ) + assert entry['outputs'], f'{target} explained no output of {key}' + own = entry['related'].get('users') + assert own is not None, f'{target} gave {key} no block of its own' + assert [int(one) for one in own['keys']] == [int(key)] + for name, block in entry['related'].items(): + # Every table of `schema` has a primary key. + assert len(block.get('keys', ())) == len(block['rows']), ( + f'{name} has a key and came back without one per row' ) + assert len(block['gradients']) == len(entry['outputs']) + for by_row in block['gradients']: + assert len(by_row) == len(block['rows']), ( + f'{name} gradients do not line up with its rows' + ) @pytest.mark.live_nim From 716f029cf7b6ba4b1875e995275fda90767fd308 Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Thu, 29 Oct 2026 14:11:29 -0700 Subject: [PATCH 14/25] Exercise a keyless related table in the live relational explanations The live fixture graph had only keyed tables, so the keyless case of the explanation contract (rows without keys, no __node_id, null key in the flat table) was never checked against a NIM. Add events, linked to users.user_id with no primary key, and check its blocks. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jana Gagacheva --- .../tests/test_live_relational.py | 66 +++++++++++++++---- 1 file changed, 53 insertions(+), 13 deletions(-) diff --git a/packages/structured-data-models-client/tests/test_live_relational.py b/packages/structured-data-models-client/tests/test_live_relational.py index 11e5a7619..e99959d4e 100644 --- a/packages/structured-data-models-client/tests/test_live_relational.py +++ b/packages/structured-data-models-client/tests/test_live_relational.py @@ -53,6 +53,12 @@ def tables() -> dict[str, pd.DataFrame]: Order counts differ sharply between the two halves, so a prediction that tracks them is evidence the whole path works rather than that something came back. + + ``events`` has no primary key: ``user_id`` repeats in it, so nothing names + one of its rows, and the engine numbers them itself. Its timestamps sit + inside the span of ``orders``, and the questions here aggregate + ``orders`` only, which is what the engine measures history and anchor + times against. """ users = pd.DataFrame( { @@ -79,7 +85,22 @@ def tables() -> dict[str, pd.DataFrame]: } ) order_id += 1 - return {'users': users, 'orders': pd.DataFrame(rows)} + events = [ + { + 'user_id': user, + 'kind': 'login' if k % 2 else 'view', + 'value': float(user % 5 + k), + 'ts': pd.Timestamp('2024-01-01') + + pd.Timedelta(days=k * 30 + (user % 9)), + } + for user in range(N_USERS) + for k in range(3) + ] + return { + 'users': users, + 'orders': pd.DataFrame(rows), + 'events': pd.DataFrame(events), + } COUNT_QUERY = 'PREDICT COUNT(orders.*, 0, 30, days) FOR EACH users.user_id' @@ -332,9 +353,12 @@ def test_run_mode_is_refused_as_an_option(query): EXPLAIN = ['prediction', 'probabilities', 'explanation'] BINARY_QUERY = 'PREDICT COUNT(orders.*, 0, 30, days) > 2 FOR EACH users.user_id' -#: Each table of ``tables()`` and its primary key. Both have one, so every -#: related block is expected to carry ``keys``; the graph has no keyless table. +#: Each keyed table of ``tables()`` and its primary key: its related blocks +#: are expected to carry ``keys``. KEYS = {'users': 'user_id', 'orders': 'order_id'} +#: The tables of ``tables()`` with no primary key: their related blocks carry +#: ``rows`` and no ``keys``. +KEYLESS = {'events'} @pytest.fixture(scope='module') @@ -349,12 +373,14 @@ def assert_explained_as_sent(result, asked): among the related ones. Each block's ``keys`` are the primary keys of its ``rows``, whose values are the caller's for that key and belong to that entity, and its gradients are ``[output][row][column]``, aligned with - ``rows``. In ``to_pandas()`` the entity's own table is at hop 0 and the - others at 1 or more. + ``rows``. A keyless table's block has ``rows`` and no ``keys``, each row + the caller's columns of one of that entity's rows. In ``to_pandas()`` the + entity's own table is at hop 0, the others at 1 or more, and a keyless + table's lines have no ``key``. """ - graph = { - name: frame.set_index(KEYS[name]) for name, frame in tables().items() - } + sent_tables = tables() + graph = {name: sent_tables[name].set_index(KEYS[name]) for name in KEYS} + keyless = {name: sent_tables[name] for name in KEYLESS} frame = result.to_pandas() assert 'explanation' not in frame.columns ids = {int(key): key for key in result.explanations} @@ -370,15 +396,24 @@ def assert_explained_as_sent(result, asked): assert outputs == 2 related = entry['related'] assert 'users' in related, 'the entity has no block of its own' - assert set(related) <= set(KEYS) + assert set(related) <= set(KEYS) | KEYLESS for name, block in related.items(): - assert 'keys' in block, f'{name} has a key and came back without' - assert len(block['keys']) == len(block['rows']) assert len(block['gradients']) == outputs for by_row in block['gradients']: assert len(by_row) == len(block['rows']) for by_column in by_row: assert len(by_column) == len(block['columns']) + if name in KEYLESS: + assert 'keys' not in block, f'{name} has no key and got keys' + own_rows = keyless[name][keyless[name]['user_id'] == entity] + for row in block['rows']: + # The caller's columns only: no instance_id, no __node_id. + assert set(row) <= set(keyless[name].columns) + assert int(row['user_id']) == entity + assert row['value'] in set(own_rows['value']) + continue + assert 'keys' in block, f'{name} has a key and came back without' + assert len(block['keys']) == len(block['rows']) for row_key, row in zip(block['keys'], block['rows'], strict=True): # The caller's columns only: no instance_id, no __node_id. assert set(row) <= {KEYS[name], *graph[name].columns} @@ -411,6 +446,9 @@ def assert_explained_as_sent(result, asked): others = flat[flat['table'] != 'users'] assert others['hop'].notna().all() assert (others['hop'] >= 1).all() + unkeyed = flat[flat['table'].isin(KEYLESS)] + assert unkeyed['key'].isna().all() + assert (unkeyed['hop'] == 1).all() @requires_relational @@ -423,8 +461,8 @@ def test_every_entity_is_explained_with_its_own_related_rows( Assumptions: - The NIM explains every predict row, both probabilities of a binary target. - - ``users`` and ``orders`` both have a primary key; the graph has no - keyless table, so the keyless case is not exercised here. + - ``users`` and ``orders`` have a primary key and ``events`` has none, + so both kinds of related block are read back. """ asked = [0, 1, 2, 3] request = binary_query.materialize(indices=asked) @@ -444,6 +482,8 @@ def test_a_request_sent_in_several_calls_explains_every_entity( related rows and keys. Assumptions: + - ``events`` has no primary key, so its rows are checked by entity and + value rather than by key. - ``rows_per_call=2`` stands in for the cap, as in the batching test above, and splits eight entities into four calls. """ From c1a0ae175ea04bc4c06edc26e53c779650e87368 Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Thu, 29 Oct 2026 14:13:06 -0700 Subject: [PATCH 15/25] Drop the server-settings caveat from the explanation docs Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jana Gagacheva --- docs/reference/limits.md | 10 +--------- docs/reference/prediction-output.md | 4 ---- skills/relational/SKILL.md | 2 +- skills/relational/references/beyond-one-answer.md | 4 +--- skills/tabular/SKILL.md | 7 ------- 5 files changed, 3 insertions(+), 24 deletions(-) diff --git a/docs/reference/limits.md b/docs/reference/limits.md index eb098458c..1e83a93bb 100644 --- a/docs/reference/limits.md +++ b/docs/reference/limits.md @@ -37,15 +37,7 @@ distinct values a column may have. Those protect a shared server, and are not applied when the model runs in your own process. **An explanation has no row limit of its own.** Every scored row is -explained, on either model. It does depend on two server settings, which a -deployment chooses when its container starts and this client cannot see: - -- one estimator, `SDM_NUM_ESTIMATORS=1`, the default; -- on `kumo-tabular`, CUDA graph replay off: `SDM_TABULAR_CUDA_GRAPHS` unset, - not `1`. - -A deployment set otherwise answers a request for `explanation` with -`422 UNSUPPORTED_OUTPUT`. Ordinary predictions are unaffected. +explained, on either model. A relational request for more entities than one call scores is not refused. `materialize()` gathers it in batches of at most that many, each with a diff --git a/docs/reference/prediction-output.md b/docs/reference/prediction-output.md index 502f1904e..dc36ae9fa 100644 --- a/docs/reference/prediction-output.md +++ b/docs/reference/prediction-output.md @@ -128,10 +128,6 @@ result = client.predict(request, model=KumoRelational(), outputs=['prediction', 'probabilities', 'explanation']) ``` -The deployment must run with one estimator, and `kumo-tabular` with CUDA graph -replay off, or it answers `422 UNSUPPORTED_OUTPUT`; see -[Limits](limits.md). - ### Keys `result.explanations` maps a row's key to that row's explanation. It is kept diff --git a/skills/relational/SKILL.md b/skills/relational/SKILL.md index 41439c02c..2062ccfb0 100644 --- a/skills/relational/SKILL.md +++ b/skills/relational/SKILL.md @@ -288,5 +288,5 @@ change. The ones worth knowing: | more than 100 calls at once | a context is gathered per call; ask about fewer entities, or raise `rows_per_call` if you lowered it | | more entities than one call scores, on a ranking or a forecast | each candidate or step is already a call, and batching one would send that many times as many; ask about fewer at a time | | an output this question cannot give | `rankings` on a regression, `explanation` on a ranking | -| `422 UNSUPPORTED_OUTPUT` | the client sent an output the deployment does not serve, such as `explanation`, or asked for `explanation` of a deployment not started with one estimator, `SDM_NUM_ESTIMATORS=1` | +| `422 UNSUPPORTED_OUTPUT` | the client sent an output the deployment does not serve, such as `explanation` | | an aggregation the model does not answer | `COUNT_DISTINCT`, `FIRST`, `LAST`, or any aggregation of a category | diff --git a/skills/relational/references/beyond-one-answer.md b/skills/relational/references/beyond-one-answer.md index dc130cb2f..7b2996b58 100644 --- a/skills/relational/references/beyond-one-answer.md +++ b/skills/relational/references/beyond-one-answer.md @@ -137,6 +137,4 @@ output, column), with `hop`, how many links the row is from the entity. The entity's own table is hop 0. A ranking refuses an explanation locally, and the shipping endpoint refuses it -everywhere else, so treat `result.explanations` as possibly empty. A deployment -that serves explanations has to run one estimator, `SDM_NUM_ESTIMATORS=1`, and -answers `422 UNSUPPORTED_OUTPUT` otherwise. +everywhere else, so treat `result.explanations` as possibly empty. diff --git a/skills/tabular/SKILL.md b/skills/tabular/SKILL.md index a2134e3de..edf4c9571 100644 --- a/skills/tabular/SKILL.md +++ b/skills/tabular/SKILL.md @@ -220,12 +220,6 @@ per unit change in that column: a diagnostic, not a causal effect or a feature-importance ranking. The whole shape is under Explanations in [`docs/reference/prediction-output.md`](../../docs/reference/prediction-output.md). -The deployment has to be started with one estimator, `SDM_NUM_ESTIMATORS=1`, -and CUDA graph replay off, `SDM_TABULAR_CUDA_GRAPHS` unset. Both are server -settings this client cannot see. A deployment started otherwise answers -`422 UNSUPPORTED_OUTPUT` to a request for `explanation`, and ordinary -predictions are unaffected. - ## When it refuses Every refusal is a `StructuredDataError` or one of its subclasses, and each @@ -235,7 +229,6 @@ names what to change. |---|---| | a task the model does not serve | `forecasting`, `temporal_link_prediction` and the rest are relational | | an output it does not produce | `rankings`, `embeddings`, `scores` | -| `422 UNSUPPORTED_OUTPUT` on `explanation` | the deployment runs more than one estimator or CUDA graph replay; see [Explanations](#explanations) | | a context above the cap | more than 10,000 labelled rows in a direct `Request` | | relationships | this model reads one table; use `kumo-relational` | From 792c29cd25c85f993c4cb8e7a7200d063e66cb3a Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Thu, 29 Oct 2026 14:14:52 -0700 Subject: [PATCH 16/25] update tracker --- gradient_xai_plan.md | 188 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 gradient_xai_plan.md diff --git a/gradient_xai_plan.md b/gradient_xai_plan.md new file mode 100644 index 000000000..59f45379e --- /dev/null +++ b/gradient_xai_plan.md @@ -0,0 +1,188 @@ +# Gradient explanations (XAI) in the client: plan and tracker + +The shared tracker for Jana, Claude and subagents. Update a row's status when it changes, and +record who decided and when. Nothing is built from a row marked NEEDS DECISION. + +**Context** +- **Client PR #34:** branch `jg-enable-explain`. Rebased onto main at `12955d397` as + `a01c08d3a` + `78b0a5ebb`, and pushed; the PR head is `78b0a5ebb`. The explanation work is + local commits on top, `793a2a8c4`..`9de1214f8`, not pushed. +- **NIM:** `structured-data-nims` origin/main `0a8230d` explains every predict row and returns one + `explanation` per prediction record. +- **Wire format** `sdm_gradient_v1`: described in the NIM repo at + `sdm_profiles/kumo_{relational,tabular}_sdm/USER_GUIDE.md`, section "Gradient Explanations". + `gradients` is `[output][row][column]`. `row_indices` are positions in the tables as submitted, + not keys. + +**Status values:** `NEEDS DECISION` (Jana chooses; the end-state column holds a proposal) · +`TO DO` (agreed, not built yet; who agreed and when, in brackets) · `WAITING (on X)` (blocked by +another row or something outside) · `DONE` (finished, with where) + +| Issue | End state / fix | Status | +|---|---|---| +| **R1** PR #34 is behind main and conflicts in `tabular.py`, `conftest.py` and `CHANGELOG.md` | Rebased onto main, conflicts resolved, the PR stays one commit; CI-equivalent checks pass locally | DONE: `a01c08d3a`; 2188 passed / 379 skipped (missing extras) / 0 failed | +| **R2** Docs said kumo-tabular doesn't explain; a test docstring stated the old `run_mode='fast'` rule as fact | Architecture capability table shows ✅; the docstring is scoped to "a deployment that…" | DONE: `78b0a5ebb` | +| **R3** The rebased branch is only local | Pushed; PR #34 shows the new commits | DONE: pushed 2026-10-28 18:59 PT, PR head `78b0a5ebb` | +| **N1** The NIM refused explanations over more than one row | The NIM explains every predict row | DONE: Jana, NIM `0a8230d` | +| **N2** Concern: a related row shared by two entities would get a mixed gradient credited to only one of them | No shared rows reach the NIM | DONE: verified by code reading plus a library test. The engine sampler copies rows per predict row, and the NIM scopes joins by `instance_id`. Not run end to end. | +| **L1** `explanation_predict_rows` in `TABULAR_LIMITS` and `RELATIONAL_LIMITS` (`models.py`) | Removed | DONE: `0da19289e` (`models.py`) | +| **L2** Tabular `_check_explanation_rows` (added by #34) | Removed, with its calls | DONE: `0da19289e` (`adapters/tabular.py`) | +| **L3** Relational one-entity explanation check (`adapters/relational.py` ~:380-400) | Removed | DONE: `0da19289e` (`adapters/relational.py`) | +| **L4** Other one-row assumptions may hide in the materializer, `decompose.py` or `client.py` | Grepped; none remain | DONE: `0da19289e`; grepped client, pql and engine, none left | +| **A1** `result.explanation` keeps only the first record's explanation and silently drops the rest | Removed. Replaced by `result.explanations`, a mapping from row key to that row's explanation, kept out of the DataFrame | DONE: `23db8a188` (`result.py` `explanations`) | +| **A2** Key for each explanation | Whatever identifies the row in `to_pandas()`: `id`; for forecasts `(id, forecast_step)`; `row_index` when the request has no entity key. The same rule for both models | DONE: `23db8a188` (`wire/explanations.py`, `client.py` `_by_step`, `_numbered_from`) | +| **A3** The instance block's `row_indices` is a position, not a key | Dropped, since the mapping key names the row; gradients become `[output][column]` | DONE: `23db8a188` | +| **A4** Related-table `row_indices` are positions in tables the caller never sees; keyless tables only carry the engine's synthetic `__node_id` (`payload.py:587-592`) | Each related row carries its primary key when the table has one, and always the row's values as sent; gradients stay `[output][row][column]`, aligned with those rows | DONE: `23db8a188` (`adapters/relational.py` `_related_rows_of`) | +| **A5** When a request is split across calls (#36), positions restart at 0 in each call | Translated inside each call's `answered()`, before merging | DONE: `23db8a188` (translated in each call's `answered()`) | +| **A7** Forecast `{'by_forecast_step': ...}` aggregation (`client.py` ~:1241-1305) | Removed; per-row keys `(id, forecast_step)` replace it | DONE: `23db8a188` (`client.py`) | +| **A9** `result.explanations` when explanations weren't requested | An empty dict | DONE: `23db8a188` | +| **A11** Type of an explanation entry | Plain dicts (A15). `result.explanations.to_pandas()` adds one flat table, with `hop` (A17). See "Reading it" in Proposed API | DONE: `23db8a188` (`Explanations.to_pandas()`) | +| **A12** `outputs` and `columns` repeat in every record | Kept per entry, as the NIM returns them | DONE: `23db8a188` | +| **A13** `columns` are the model's preprocessed columns; categoricals and datetimes appear as derived columns | Passed through as the NIM returns them, and documented (D3) | DONE: passed through; documented in `bff621aba` | +| **A14** `__contains__`, has-field and repr refer to `explanation` | Updated for `explanations` | DONE: `23db8a188` | +| **A15** Shape of one entry | `{'outputs': [...], 'instance': {'columns', 'gradients' [output][column]}, 'related': {table: {'keys', 'rows' (values as sent), 'columns', 'gradients' [output][row][column]}}}` | DONE: `23db8a188`, `ca99c782f` | +| **A16** Each model's entry has only the tables the caller sent. kumo-tabular has no related tables. For kumo-relational the client's engine sends the entity as two tables: a question row (`instance_table`: `instance_id`, entity reference, anchor time; no caller columns) and the entity's own table inside `related_tables`, and the NIM explains both | kumo-tabular entries have no `related` key. kumo-relational entries have no `instance` key: the client drops the question row's block when it reads the reply, and the entity's own table appears under `related` like every other table, at hop 0 | DONE: `ca99c782f` (relational entries have no `instance`) | +| **A17** The `hop` column in `to_pandas()` (A11) needs the client to work out each related row's distance from the entity | Computed from the relationships and foreign-key values in the payload sent, per call; checked against a real multi-hop payload | DONE: `23db8a188` (`_hops_from_the_entity`); checked on the golden one-hop payload; deeper hops are checked by T4 | +| **B2** The `live_nim` parity case in `test_every_backend_answers_alike.py` expects a NIM to refuse relational explanations | The case is removed; T1 covers both behaviours (the in-process backend refuses, the NIM answers) | DONE: `b2a30cd3e` | +| **T1** Tests for the new behaviour | Written per the `unittesting` skill. Cover tabular multi-row; relational multi-row with key and value translation; keyless related table; a split request whose second call restarts at 0; forecast keys; the in-process backend refusing `'explanation'`; `to_pandas()` | DONE: `a79c00aa0`, `ca99c782f` (unit tests with mocked NIM replies) | +| **T2** Old tests assert the one-row refusals and `result.explanation` | Updated or removed | DONE: `0da19289e`, `23db8a188`, `36de54755` | +| **T3** Local checks | ruff, format, mypy, hygiene and the CI pytest command all pass; counts recorded here | DONE: ruff, format, mypy (184 files), hygiene, client suite 1580 passed / 256 skipped / 0 failed, root tests 174 passed / 109 skipped (2026-10-29, rerun by the main agent) | +| **T4** The client has never run against a NIM that has `0a8230d` | Explanation cases (tabular and relational, multi-row, keyless related table, a split request) added to `test_live_nim.py` and `test_live_relational.py`, run once against a NIM built from `0a8230d`; output checked against A1-A5 | WAITING (on the rebuilt NIM image; Jana names the endpoint). Test code written: `9de1214f8` (`test_live_nim.py`, `test_live_relational.py`, `tests/test_skills.py`). Keyless related table added to the live fixture: `716f029cf` (agreed: Jana, 2026-10-29). Also at T4 time: add `explanation` to `SERVED` in `tests/test_skills.py` so the skill-matrix live test stops expecting a refusal (agreed: Jana, 2026-10-29). Both images triggered in the NIM repo CI (`SDM_PROFILE=both`) 2026-10-29 | +| **D1** `result.py` module docstring shows `result.explanation` | Shows `result.explanations` and its keys | DONE: `23db8a188` | +| **D2** `docs/reference/limits.md` explanation rows | The row limit is removed | DONE: `793a2a8c4` | +| **D3** `docs/reference/prediction-output.md` | Has a section on explanations: shape, keys, related rows, preprocessed columns | DONE: `793a2a8c4`, `964ded993`, `bff621aba` | +| **D4** `docs/about/architecture.md` | Explains the explanation design and why (from this file) | DONE: `793a2a8c4`, `bff621aba` | +| **D5** `skills/tabular/SKILL.md` and `skills/relational/SKILL.md` say one row per explanation | Multi-row, `result.explanations`, with an example | DONE: `793a2a8c4`, `bff621aba` | +| **D6** `qa/probe_tabular.py` | Probes a multi-row explanation | DONE: `793a2a8c4`; `qa/probe_relational.py` too (`964ded993`, `bff621aba`); not run, no NIM | +| **V1** CHANGELOG `## Unreleased` | Changed: multi-row explanations. Removed: `result.explanation` and `by_forecast_step`. Added: `result.explanations` | DONE: `793a2a8c4`, `bff621aba` | + +## Proposed API + +The IDs in brackets refer to rows in the tracker table above. Each item's status lives only +there. + +### Asking + +Asking is unchanged: add `'explanation'` to `outputs`. There is no row limit (L1-L3). + +```python +result = client.predict(request, model=KumoRelational(), + outputs=['prediction', 'probabilities', 'explanation']) +``` + +### `result.explanations` + +`result.explanations` is a mapping from a row's key to that row's explanation. It is kept out of +`result.to_pandas()` (A1). `result.explanation` and `{'by_forecast_step': ...}` no longer exist +(A1, A7). + +| Request | Key | Example | +|---|---|---| +| has an entity id | the row's `id`, the same value as in `to_pandas()['id']` | `result.explanations['user_5']` | +| forecast | `(id, forecast_step)` | `result.explanations[('user_5', 3)]` | +| no entity id (single-table tabular) | `row_index` (A2) | `result.explanations[0]` | + +### One entry (A15) + +```python +{ + 'outputs': [...], # the explained scores, in order (see below) + 'instance': { + 'columns': [...], # model's preprocessed numerical columns (A13) + 'gradients': [[...], ...], # [output][column] + }, + 'related': { # absent for kumo-tabular; for kumo-relational it + # holds every table incl. the entity's own, and + # there is no 'instance' (A16) + '
': { + 'keys': [...], # the table's primary key per row; absent if the table has none + 'rows': [{...}, ...], # each row's values as sent to the model + 'columns': [...], # model's preprocessed numerical columns (A13) + 'gradients': [[[...], ...], ...], # [output][row][column], rows aligned with keys and rows + }, + }, +} +``` + +`outputs` is copied from the NIM: +- classification gives every class probability in declared order, e.g. + `{'kind': 'probability', 'class': 'yes'}`; +- regression gives the returned prediction, `{'kind': 'prediction', 'statistic': 'mean'}` + (or `'median'`), plus each returned quantile when `quantiles` is requested. + +A gradient is the change in that output score per unit change in that preprocessed column. It is +a diagnostic, not a causal effect or a feature-importance ranking. + +### Which tables get gradients + +| Model | Tables with gradients | +|---|---| +| kumo-tabular | the one table: each predicted row's own columns | +| kumo-relational | the predicted entity's own table, plus every related table in the subgraph the client gathered for that entity (all tables reached within the hop limit, up to 6). The engine's question row (`instance_table`) is explained by the NIM too, but the client drops that block (A16) | + +- Only the rows being predicted and their related rows get gradients. The context rows (the + labelled examples the model compares against) get none (`EnableInputGradients` in + `sdm/explain/gradient.py`). +- Only numerical inputs as the model sees them get gradients. Categoricals and datetimes appear as + derived numeric columns; a table with none has an empty `columns` (A13). +- Gradients are per row, not per path. A row reached through several paths, for example a product + bought in two of the user's orders, appears once. Its gradient is its total effect through all + of them, and the model gives no split by path. + +### Examples (values illustrative) + +kumo-tabular, request with an id column: + +```python +result.explanations['cust_17'] +# {'outputs': [{'kind': 'probability', 'class': 'no'}, +# {'kind': 'probability', 'class': 'yes'}], +# 'instance': {'columns': ['age', 'balance'], +# 'gradients': [[0.01, -0.03], [-0.01, 0.03]]}} +``` + +kumo-relational, user 5 with one order and two login events. `orders` has a primary key +(`order_id`), so its rows carry `keys`. `events` has none, so its rows are identified by their +values alone. Both are part of the same entry: + +```python +result.explanations['user_5'] +# {'outputs': [{'kind': 'probability', 'class': 'no'}, +# {'kind': 'probability', 'class': 'yes'}], +# 'related': { +# 'users': {'keys': ['user_5'], +# 'rows': [{'user_id': 'user_5', 'tenure_days': 410}], +# 'columns': ['tenure_days'], +# 'gradients': [[[-0.05]], [[0.05]]]}, +# 'orders': {'keys': ['order_9003'], +# 'rows': [{'user_id': 'user_5', 'order_id': 'order_9003', 'amount': 12.0}], +# 'columns': ['amount'], +# 'gradients': [[[-0.02]], [[0.02]]]}, +# 'events': {'rows': [{'user_id': 'user_5', 'kind': 'login', 'at': '2026-10-01T09:00:00'}, +# {'user_id': 'user_5', 'kind': 'login', 'at': '2026-10-03T18:20:00'}], +# 'columns': ['kind_login', 'at_dayofweek'], # model-derived names, illustrative (A13) +# 'gradients': [[[0.01, 0.00], [0.02, -0.01]], +# [[-0.01, 0.00], [-0.02, 0.01]]]}}} +``` + +Keyed or keyless changes only how a row is labelled in the output. Both tables get gradients in +the same way. + +### Reading it (A11) + +Each entry is the plain nested dict shown above. `result.explanations.to_pandas()` gives one line +per (row, output, column), across all entities: + +``` +id table hop key row output column gradient +user_5 users 0 user_5 0 probability_yes tenure_days 0.05 +user_5 orders 1 order_9003 0 probability_yes amount 0.02 +user_5 products 2 prod_77 0 probability_yes price 0.004 +user_5 events 1 None 1 probability_yes kind_login -0.02 +``` + +- `hop` is how many links the row is from the entity (0 is the entity's own table; for kumo-tabular the one table, shown as `instance`). `None` for a row no relationship reaches. The client works + it out from the relationships and foreign-key values it sent (A17). +- `output` is the matching `result.to_pandas()` column name (`probability_yes`, `prediction`, `quantile_0.1`). +- `key` is the row's primary key, or `None` for a keyless table. `row` is the row's position in + that entity's `related[table]['rows']`. From eab04455733e7c3656eee63692657bddae08fa61 Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Thu, 29 Oct 2026 14:37:09 -0700 Subject: [PATCH 17/25] Read explanations through a GradientDecoder Rename the result class Explanations to GradientExplanations, and move explanations_of and the relational adapter's related-row helpers into one GradientDecoder in wire/explanations.py, built once per call with the call's payload (kumo-relational) or none (kumo-tabular). No behaviour change. The public API snapshot is regenerated for the rename. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jana Gagacheva --- .../src/sdm_client/__init__.py | 4 +- .../src/sdm_client/adapters/relational.py | 179 +---------- .../src/sdm_client/adapters/tabular.py | 4 +- .../src/sdm_client/client.py | 4 +- .../src/sdm_client/result.py | 24 +- .../src/sdm_client/wire/explanations.py | 281 +++++++++++++++--- tests/golden/public_api.json | 24 +- 7 files changed, 266 insertions(+), 254 deletions(-) diff --git a/packages/structured-data-models-client/src/sdm_client/__init__.py b/packages/structured-data-models-client/src/sdm_client/__init__.py index 561991ba2..34f155da7 100644 --- a/packages/structured-data-models-client/src/sdm_client/__init__.py +++ b/packages/structured-data-models-client/src/sdm_client/__init__.py @@ -33,7 +33,7 @@ from sdm_client.models import KumoRelational, KumoTabular, Model from sdm_client.query import PredictiveQuery from sdm_client.request import Request, Split, Target, Task -from sdm_client.result import Explanations, PredictionResult +from sdm_client.result import GradientExplanations, PredictionResult from sdm_client.schema import ( Column, Relationship, @@ -45,8 +45,8 @@ __all__ = [ 'SDMWarning', 'Column', - 'Explanations', 'FittedModel', + 'GradientExplanations', 'KumoRelational', 'KumoTabular', 'MissingExtraError', diff --git a/packages/structured-data-models-client/src/sdm_client/adapters/relational.py b/packages/structured-data-models-client/src/sdm_client/adapters/relational.py index 93bb939e5..723bbc360 100644 --- a/packages/structured-data-models-client/src/sdm_client/adapters/relational.py +++ b/packages/structured-data-models-client/src/sdm_client/adapters/relational.py @@ -49,7 +49,7 @@ from sdm_client.result import PredictionResult from sdm_client.wire import as_declared from sdm_client.wire.answers import as_declared_target -from sdm_client.wire.explanations import RelatedReader, explanations_of +from sdm_client.wire.explanations import GradientDecoder from sdm_client.wire.inference import typed_inference # What to ask for when the caller named nothing, by task kind. Asking for @@ -292,9 +292,7 @@ def answered( outputs=outputs, # Read against this call's own tables, whose positions restart at # 0 in every call of a split request. - explanations=explanations_of( - named, _related_rows_of(payload or {}) - ), + explanations=GradientDecoder(payload or {}).decode(named), ) def explain_error( @@ -825,176 +823,3 @@ def _key_dtype(payload: dict[str, Any], reference: str) -> str | None: .get('dtype') ) return None - - -#: The engine's own transport columns on a related table: the instance a row -#: was sampled for, and the key it numbers a keyless table's rows by -#: (`rfm/inference/payload.py`). Neither is a value the caller sent, so a -#: related row is given back without them. -_INSTANCE_ID = 'instance_id' -_SYNTHETIC_KEY = '__node_id' - - -def _related_rows_of(payload: Mapping[str, Any]) -> RelatedReader: - r"""Read a record's related blocks as rows of this call, not positions. - - The endpoint names each related row by its position in the table as this - call sent it, which the caller never saw and which a split request - restarts at 0 in every call. Each is given back as its primary key, when - the table has one of the caller's, and always as its values as sent. - - How far each row is from the entity is worked out once per call, and only - if some record is explained. - """ - schema = payload.get('schema') or {} - sent = (payload.get('predict') or {}).get('related_tables') or {} - hops: dict[str, list[int | None]] | None = None - - def read( - blocks: Mapping[str, Any], - ) -> tuple[dict[str, Any], dict[str, list[int | None]]]: - nonlocal hops - if hops is None: - hops = _hops_from_the_entity(payload) - related: dict[str, Any] = {} - away: dict[str, list[int | None]] = {} - for name, block in blocks.items(): - table = sent.get(name) or {} - columns = list(table.get('columns') or []) - rows = [table['rows'][index] for index in block['row_indices']] - described = (schema.get('related_tables') or {}).get(name) or {} - key = _caller_key(described) - hidden = {_INSTANCE_ID, *_synthetic_keys(described)} - read_back: dict[str, Any] = {} - if key is not None and key in columns: - dtype = ( - (described.get('columns') or {}).get(key, {}).get('dtype') - ) - read_back['keys'] = [ - as_declared(row[columns.index(key)], dtype) for row in rows - ] - read_back['rows'] = [ - { - column: value - for column, value in zip(columns, row, strict=True) - if column not in hidden - } - for row in rows - ] - read_back['columns'] = list(block.get('columns') or []) - read_back['gradients'] = block.get('gradients') or [] - related[name] = read_back - away[name] = [hops[name][index] for index in block['row_indices']] - return related, away - - return read - - -def _key_columns(described: Mapping[str, Any]) -> list[str]: - r"""A related table's primary key, less the instance that scopes it.""" - key = described.get('primary_key') or [] - key = [key] if isinstance(key, str) else list(key) - return [column for column in key if column != _INSTANCE_ID] - - -def _synthetic_keys(described: Mapping[str, Any]) -> list[str]: - r"""The key columns the engine made up for a table that had none.""" - return [ - column - for column in _key_columns(described) - if column.startswith(_SYNTHETIC_KEY) - ] - - -def _caller_key(described: Mapping[str, Any]) -> str | None: - r"""The table's own primary key, or ``None`` if it has none of its own.""" - key = _key_columns(described) - if len(key) != 1 or _synthetic_keys(described): - return None - return key[0] - - -def _hops_from_the_entity( - payload: Mapping[str, Any], -) -> dict[str, list[int | None]]: - r"""How many links each related row of this call is from its entity. - - Read from the relationships and key values sent, the same ones the model - joins on. A relationship with no ``source_table`` starts at the instance - table, and the rows it reaches are the entity's own: 0. Every other one is - followed either way, and a row is as far as its nearest path. Joins stay - within one instance, as the engine scopes them, so two entities sharing a - product never meet through it. - - Positions are this call's, one list per related table of its predict half. - ``None`` for a row no relationship reaches from an entity. - """ - schema = payload.get('schema') or {} - predict = payload.get('predict') or {} - instance = predict.get('instance_table') or {} - tables = predict.get('related_tables') or {} - hops: dict[str, list[int | None]] = { - name: [None] * len(table.get('rows') or []) - for name, table in tables.items() - } - linked: dict[tuple[str, int], list[tuple[str, int]]] = {} - reached: list[tuple[str, int]] = [] - for link in schema.get('relationships') or []: - target = link.get('target_table') - if target not in tables: - continue - by_value: dict[Any, list[int]] = {} - for position, value in enumerate( - _joined_on(tables[target], link.get('target_columns') or []) - ): - if value is not None: - by_value.setdefault(value, []).append(position) - source = link.get('source_table') - columns = link.get('source_columns') or [] - if source is None: - for value in _joined_on(instance, columns): - reached += [(target, at) for at in by_value.get(value, ())] - continue - if source not in tables: - continue - for position, value in enumerate(_joined_on(tables[source], columns)): - for at in by_value.get(value, ()): - linked.setdefault((source, position), []).append((target, at)) - linked.setdefault((target, at), []).append((source, position)) - - for name, position in reached: - hops[name][position] = 0 - frontier = reached - distance = 0 - while frontier: - distance += 1 - following = [] - for row in frontier: - for name, position in linked.get(row, ()): - if hops[name][position] is None: - hops[name][position] = distance - following.append((name, position)) - frontier = following - return hops - - -def _joined_on(table: Mapping[str, Any], columns: Sequence[str]) -> list[Any]: - r"""What each row of ``table`` joins on through ``columns``. - - The instance a row belongs to, and its values in ``columns``; ``None`` for - a row holding a null there, which joins nothing. Empty when the table does - not carry every one of ``columns``. - """ - names = list(table.get('columns') or []) - if not columns or any(column not in names for column in columns): - return [] - at = [names.index(column) for column in columns] - scope = names.index(_INSTANCE_ID) if _INSTANCE_ID in names else None - joined: list[Any] = [] - for row in table.get('rows') or []: - values = tuple(row[index] for index in at) - if any(value is None for value in values): - joined.append(None) - continue - joined.append((None if scope is None else row[scope], values)) - return joined diff --git a/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py b/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py index dd4d693a0..6d693b58e 100644 --- a/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py +++ b/packages/structured-data-models-client/src/sdm_client/adapters/tabular.py @@ -50,7 +50,7 @@ widen_tfm_dtype, ) from sdm_client.wire.answers import as_declared_target -from sdm_client.wire.explanations import explanations_of +from sdm_client.wire.explanations import GradientDecoder from sdm_client.wire.inference import typed_inference # Output fields to ask for when the caller named none, by task kind. Asking for @@ -172,7 +172,7 @@ def answered( metadata=dict(metadata or {}), outputs=outputs, # One table, so an entry has no related rows to name. - explanations=explanations_of(named), + explanations=GradientDecoder().decode(named), ) def explain_error( diff --git a/packages/structured-data-models-client/src/sdm_client/client.py b/packages/structured-data-models-client/src/sdm_client/client.py index ab5c20fec..fae7d9111 100644 --- a/packages/structured-data-models-client/src/sdm_client/client.py +++ b/packages/structured-data-models-client/src/sdm_client/client.py @@ -48,7 +48,7 @@ _envelope_rows, warn_of_rows_scored_from_their_own_answers, ) -from sdm_client.result import Explanations, PredictionResult +from sdm_client.result import GradientExplanations, PredictionResult from sdm_client.schema import _as_key_tuple from sdm_client.wire.keys import ( _named_by, @@ -1294,7 +1294,7 @@ def _by_step( for (_, step), answer in zip(rewritten.parts, answers, strict=True) for item in answer.items ] - explanations = Explanations._merged( + explanations = GradientExplanations._merged( [ answer.explanations._at_step(step) for (_, step), answer in zip(rewritten.parts, answers, strict=True) diff --git a/packages/structured-data-models-client/src/sdm_client/result.py b/packages/structured-data-models-client/src/sdm_client/result.py index cf0d21780..a8ce229b1 100644 --- a/packages/structured-data-models-client/src/sdm_client/result.py +++ b/packages/structured-data-models-client/src/sdm_client/result.py @@ -62,7 +62,7 @@ class as columns rather than a row per class, while forecasting gives one row 'embeddings', ) -# The columns of `Explanations.to_pandas()` after the ones naming the row. +# The columns of `GradientExplanations.to_pandas()` after the ones naming the row. _GRADIENT_COLUMNS = ( 'table', 'hop', @@ -74,7 +74,7 @@ class as columns rather than a row per class, while forecasting gives one row ) -class Explanations(dict[Any, dict[str, Any]]): +class GradientExplanations(dict[Any, dict[str, Any]]): r"""Each explained row's explanation, under the key naming it in the frame. The key is the row's ``id``, ``(id, forecast_step)`` for a forecast, and @@ -180,29 +180,31 @@ def to_pandas(self) -> pd.DataFrame: lines, columns=[*self._keyed_by, *_GRADIENT_COLUMNS], dtype=object ).astype({'hop': 'Int64', 'row': 'int64', 'gradient': 'float64'}) - def _renumbered(self, start: int) -> Explanations: + def _renumbered(self, start: int) -> GradientExplanations: r"""These, with each ``row_index`` key ``start`` places further on. A key that is an ``id`` is not a position and is left alone. """ if not start or self._keyed_by != ('row_index',): return self - return Explanations( + return GradientExplanations( {key + start: entry for key, entry in self.items()}, keyed_by=self._keyed_by, hops={key + start: hops for key, hops in self._hops.items()}, ) - def _at_step(self, step: Any) -> Explanations: + def _at_step(self, step: Any) -> GradientExplanations: r"""These, as the explanations of one forecast step.""" - return Explanations( + return GradientExplanations( {(key, step): entry for key, entry in self.items()}, keyed_by=(*self._keyed_by, 'forecast_step'), hops={(key, step): hops for key, hops in self._hops.items()}, ) @classmethod - def _merged(cls, parts: Sequence[Explanations]) -> Explanations: + def _merged( + cls, parts: Sequence[GradientExplanations] + ) -> GradientExplanations: r"""The explanations of several calls, as one. Each call answers about its own rows, so no key is in two parts. @@ -274,8 +276,8 @@ def __init__( self.metadata = dict(metadata or {}) self._explanations = ( explanations - if isinstance(explanations, Explanations) - else Explanations(explanations) + if isinstance(explanations, GradientExplanations) + else GradientExplanations(explanations) ) self._unscored_candidates = tuple(unscored_candidates) @@ -317,7 +319,7 @@ def _carries(self, name: str) -> bool: ) @property - def explanations(self) -> Explanations: + def explanations(self) -> GradientExplanations: r"""Each explained row's explanation, under the key naming that row. The key is the one ``to_pandas()`` names the row by: its ``id``, @@ -458,7 +460,7 @@ def merge(cls, parts: Sequence[PredictionResult]) -> PredictionResult: # would otherwise survive. That is the one you do not have when a # later batch is the one that misbehaved. metadata['batch_request_ids'] = [part.request_id for part in parts] - explanations = Explanations._merged( + explanations = GradientExplanations._merged( [part.explanations for part in parts] ) # Every part answers one slice of the same request, so they were asked diff --git a/packages/structured-data-models-client/src/sdm_client/wire/explanations.py b/packages/structured-data-models-client/src/sdm_client/wire/explanations.py index 646b90a27..79b9fc5a4 100644 --- a/packages/structured-data-models-client/src/sdm_client/wire/explanations.py +++ b/packages/structured-data-models-client/src/sdm_client/wire/explanations.py @@ -17,58 +17,243 @@ from __future__ import annotations -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Mapping, Sequence from typing import Any -from sdm_client.result import Explanations +from sdm_client.result import GradientExplanations +from sdm_client.wire.dtypes import as_declared -#: Reads one record's ``related_tables`` into the entry's ``related``, and -#: says how many links each of those rows is from the entity. -RelatedReader = Callable[ - [Mapping[str, Any]], - tuple[dict[str, Any], dict[str, list[int | None]]], -] +class GradientDecoder: + r"""Reads one call's explanations, beside that call's own payload. -def explanations_of( - items: Sequence[Any], related: RelatedReader | None = None -) -> Explanations: - r"""Each answered row's explanation, under the key naming that row. - - ``items`` have their ``id`` restored already, so the key is the one the - frame shows: the ``id``, or the ``row_index`` when the rows carry none. - ``related`` is given for a model that reads related tables. Then an entry - has ``related`` and no ``instance``: that model's ``instance_table`` is the - question row its engine sends, not the caller's, and the entity's own table - is among the related ones. Without it an entry has ``instance`` and no - ``related`` at all, rather than an empty one. + Constructed once per call: with the payload for ``kumo-relational``, whose + explanations name rows of the related tables it sent, and with none for + ``kumo-tabular``, whose explanations are about the record's own row only. """ - explained = [item for item in items if item.explanation] - if not explained: - return Explanations() - by_id = any(item.id is not None for item in explained) - entries: dict[Any, dict[str, Any]] = {} - hops: dict[Any, dict[str, list[int | None]]] = {} - for item in explained: - key = item.id if by_id else item.row_index - details = item.explanation.get('details') or {} - entry: dict[str, Any] = {'outputs': list(details.get('outputs') or [])} - if related is None: - instance = details.get('instance_table') or {} - entry['instance'] = { - 'columns': list(instance.get('columns') or []), - # One row, the record's own, so the row axis says nothing. - 'gradients': [ - by_row[0] for by_row in instance.get('gradients') or [] - ], + + #: The engine's own transport columns on a related table: the instance a + #: row was sampled for, and the key it numbers a keyless table's rows by + #: (`rfm/inference/payload.py`). Neither is a value the caller sent, so a + #: related row is given back without them. + INSTANCE_ID = 'instance_id' + SYNTHETIC_KEY = '__node_id' + + def __init__(self, payload: Mapping[str, Any] | None = None) -> None: + self._payload = payload + # How far each related row is from the entity, worked out once per + # call, and only if some record is explained. + self._hops: dict[str, list[int | None]] | None = None + + def decode(self, items: Sequence[Any]) -> GradientExplanations: + r"""Each answered row's explanation, under the key naming that row. + + ``items`` have their ``id`` restored already, so the key is the one the + frame shows: the ``id``, or the ``row_index`` when the rows carry none. + A payload is given for a model that reads related tables. Then an entry + has ``related`` and no ``instance``: that model's ``instance_table`` is + the question row its engine sends, not the caller's, and the entity's + own table is among the related ones. Without it an entry has + ``instance`` and no ``related`` at all, rather than an empty one. + """ + explained = [item for item in items if item.explanation] + if not explained: + return GradientExplanations() + by_id = any(item.id is not None for item in explained) + entries: dict[Any, dict[str, Any]] = {} + hops: dict[Any, dict[str, list[int | None]]] = {} + for item in explained: + key = item.id if by_id else item.row_index + details = item.explanation.get('details') or {} + entry: dict[str, Any] = { + 'outputs': list(details.get('outputs') or []) } - else: - entry['related'], hops[key] = related( - details.get('related_tables') or {} - ) - entries[key] = entry - return Explanations( - entries, - keyed_by=('id',) if by_id else ('row_index',), - hops=hops, - ) + if self._payload is None: + instance = details.get('instance_table') or {} + entry['instance'] = { + 'columns': list(instance.get('columns') or []), + # One row, the record's own, so the row axis says nothing. + 'gradients': [ + by_row[0] for by_row in instance.get('gradients') or [] + ], + } + else: + entry['related'], hops[key] = self._read_related( + self._payload, details.get('related_tables') or {} + ) + entries[key] = entry + return GradientExplanations( + entries, + keyed_by=('id',) if by_id else ('row_index',), + hops=hops, + ) + + def _read_related( + self, payload: Mapping[str, Any], blocks: Mapping[str, Any] + ) -> tuple[dict[str, Any], dict[str, list[int | None]]]: + r"""Read a record's related blocks as rows of this call, not positions. + + The endpoint names each related row by its position in the table as + this call sent it, which the caller never saw and which a split request + restarts at 0 in every call. Each is given back as its primary key, + when the table has one of the caller's, and always as its values as + sent. + + How far each row is from the entity is worked out once per call, and + only if some record is explained. + """ + schema = payload.get('schema') or {} + sent = (payload.get('predict') or {}).get('related_tables') or {} + if self._hops is None: + self._hops = self._hops_from_entity(payload) + hops = self._hops + related: dict[str, Any] = {} + away: dict[str, list[int | None]] = {} + for name, block in blocks.items(): + table = sent.get(name) or {} + columns = list(table.get('columns') or []) + rows = [table['rows'][index] for index in block['row_indices']] + described = (schema.get('related_tables') or {}).get(name) or {} + key = self._caller_key(described) + hidden = {self.INSTANCE_ID, *self._synthetic_keys(described)} + read_back: dict[str, Any] = {} + if key is not None and key in columns: + dtype = ( + (described.get('columns') or {}).get(key, {}).get('dtype') + ) + read_back['keys'] = [ + as_declared(row[columns.index(key)], dtype) for row in rows + ] + read_back['rows'] = [ + { + column: value + for column, value in zip(columns, row, strict=True) + if column not in hidden + } + for row in rows + ] + read_back['columns'] = list(block.get('columns') or []) + read_back['gradients'] = block.get('gradients') or [] + related[name] = read_back + away[name] = [hops[name][index] for index in block['row_indices']] + return related, away + + @classmethod + def _key_columns(cls, described: Mapping[str, Any]) -> list[str]: + r"""A related table's primary key, less the instance that scopes it.""" + key = described.get('primary_key') or [] + key = [key] if isinstance(key, str) else list(key) + return [column for column in key if column != cls.INSTANCE_ID] + + @classmethod + def _synthetic_keys(cls, described: Mapping[str, Any]) -> list[str]: + r"""The key columns the engine made up for a table that had none.""" + return [ + column + for column in cls._key_columns(described) + if column.startswith(cls.SYNTHETIC_KEY) + ] + + @classmethod + def _caller_key(cls, described: Mapping[str, Any]) -> str | None: + r"""The table's own primary key, or ``None`` if it has none of its own.""" + key = cls._key_columns(described) + if len(key) != 1 or cls._synthetic_keys(described): + return None + return key[0] + + @classmethod + def _hops_from_entity( + cls, payload: Mapping[str, Any] + ) -> dict[str, list[int | None]]: + r"""How many links each related row of this call is from its entity. + + Read from the relationships and key values sent, the same ones the + model joins on. A relationship with no ``source_table`` starts at the + instance table, and the rows it reaches are the entity's own: 0. Every + other one is followed either way, and a row is as far as its nearest + path. Joins stay within one instance, as the engine scopes them, so two + entities sharing a product never meet through it. + + Positions are this call's, one list per related table of its predict + half. ``None`` for a row no relationship reaches from an entity. + """ + schema = payload.get('schema') or {} + predict = payload.get('predict') or {} + instance = predict.get('instance_table') or {} + tables = predict.get('related_tables') or {} + hops: dict[str, list[int | None]] = { + name: [None] * len(table.get('rows') or []) + for name, table in tables.items() + } + linked: dict[tuple[str, int], list[tuple[str, int]]] = {} + reached: list[tuple[str, int]] = [] + for link in schema.get('relationships') or []: + target = link.get('target_table') + if target not in tables: + continue + by_value: dict[Any, list[int]] = {} + for position, value in enumerate( + cls._joined_on(tables[target], link.get('target_columns') or []) + ): + if value is not None: + by_value.setdefault(value, []).append(position) + source = link.get('source_table') + columns = link.get('source_columns') or [] + if source is None: + for value in cls._joined_on(instance, columns): + reached += [(target, at) for at in by_value.get(value, ())] + continue + if source not in tables: + continue + for position, value in enumerate( + cls._joined_on(tables[source], columns) + ): + for at in by_value.get(value, ()): + linked.setdefault((source, position), []).append( + (target, at) + ) + linked.setdefault((target, at), []).append( + (source, position) + ) + + for name, position in reached: + hops[name][position] = 0 + frontier = reached + distance = 0 + while frontier: + distance += 1 + following = [] + for row in frontier: + for name, position in linked.get(row, ()): + if hops[name][position] is None: + hops[name][position] = distance + following.append((name, position)) + frontier = following + return hops + + @classmethod + def _joined_on( + cls, table: Mapping[str, Any], columns: Sequence[str] + ) -> list[Any]: + r"""What each row of ``table`` joins on through ``columns``. + + The instance a row belongs to, and its values in ``columns``; ``None`` + for a row holding a null there, which joins nothing. Empty when the + table does not carry every one of ``columns``. + """ + names = list(table.get('columns') or []) + if not columns or any(column not in names for column in columns): + return [] + at = [names.index(column) for column in columns] + scope = ( + names.index(cls.INSTANCE_ID) if cls.INSTANCE_ID in names else None + ) + joined: list[Any] = [] + for row in table.get('rows') or []: + values = tuple(row[index] for index in at) + if any(value is None for value in values): + joined.append(None) + continue + joined.append((None if scope is None else row[scope], values)) + return joined diff --git a/tests/golden/public_api.json b/tests/golden/public_api.json index eb77f0a0d..8b09df16c 100644 --- a/tests/golden/public_api.json +++ b/tests/golden/public_api.json @@ -112,8 +112,8 @@ "sdm_client": { "__all__": [ "Column", - "Explanations", "FittedModel", + "GradientExplanations", "KumoRelational", "KumoTabular", "MissingExtraError", @@ -150,17 +150,6 @@ }, "signature": "(name: 'str', dtype: 'str', stype: 'str') -> None" }, - "Explanations": { - "bases": [ - "builtins.dict" - ], - "kind": "class", - "public_attributes": [], - "public_methods": { - "to_pandas": "(self) -> 'pd.DataFrame'" - }, - "signature": "(entries: 'Mapping[Any, dict[str, Any]] | None' = None, *, keyed_by: 'tuple[str, ...]' = ('id',), hops: 'Mapping[Any, Mapping[str, Sequence[int | None]]] | None' = None) -> 'None'" - }, "FittedModel": { "bases": [], "kind": "class", @@ -174,6 +163,17 @@ }, "signature": "() -> 'None'" }, + "GradientExplanations": { + "bases": [ + "builtins.dict" + ], + "kind": "class", + "public_attributes": [], + "public_methods": { + "to_pandas": "(self) -> 'pd.DataFrame'" + }, + "signature": "(entries: 'Mapping[Any, dict[str, Any]] | None' = None, *, keyed_by: 'tuple[str, ...]' = ('id',), hops: 'Mapping[Any, Mapping[str, Sequence[int | None]]] | None' = None) -> 'None'" + }, "KumoRelational": { "bases": [ "sdm_client.models.Model" From 83b12dd49e6d3c37f2ea8a7053ce148a826da2b2 Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Thu, 29 Oct 2026 14:38:47 -0700 Subject: [PATCH 18/25] Trim the explanation decoder's docstrings to one line Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jana Gagacheva --- .../src/sdm_client/result.py | 22 +----- .../src/sdm_client/wire/explanations.py | 72 +++---------------- 2 files changed, 12 insertions(+), 82 deletions(-) diff --git a/packages/structured-data-models-client/src/sdm_client/result.py b/packages/structured-data-models-client/src/sdm_client/result.py index a8ce229b1..bb3406828 100644 --- a/packages/structured-data-models-client/src/sdm_client/result.py +++ b/packages/structured-data-models-client/src/sdm_client/result.py @@ -75,27 +75,7 @@ class as columns rather than a row per class, while forecasting gives one row class GradientExplanations(dict[Any, dict[str, Any]]): - r"""Each explained row's explanation, under the key naming it in the frame. - - The key is the row's ``id``, ``(id, forecast_step)`` for a forecast, and - its ``row_index`` when the request has no entity key. An entry is a plain - dict:: - - {'outputs': [...], # the explained scores - 'instance': {'columns': [...], # kumo-tabular only - 'gradients': [...]}, # [output][column] - 'related': {table: {'keys': [...], # absent for a keyless table - 'rows': [{...}], # each row's values as sent - 'columns': [...], - 'gradients': [...]}}} # [output][row][column] - - ``instance`` is there for ``kumo-tabular`` only, and ``related`` for - ``kumo-relational`` only, where it holds every table, the entity's own - among them. ``columns`` are the model's preprocessed numerical columns, as - the endpoint names them. - - Empty when no explanation was asked for, or none came back. - """ + r"""Each explained row's gradient explanation, keyed as the row is in the frame.""" def __init__( self, diff --git a/packages/structured-data-models-client/src/sdm_client/wire/explanations.py b/packages/structured-data-models-client/src/sdm_client/wire/explanations.py index 79b9fc5a4..9c7231b9e 100644 --- a/packages/structured-data-models-client/src/sdm_client/wire/explanations.py +++ b/packages/structured-data-models-client/src/sdm_client/wire/explanations.py @@ -2,18 +2,7 @@ # All rights reserved. # SPDX-License-Identifier: Apache-2.0 -r"""Reading the explanation each prediction record carries. - -Both models answer an explanation in one format, ``sdm_gradient_v1``, one per -record, about that record's own row. Its blocks name rows by their position in -the tables of the call that produced it, and a request sent in several calls -starts those positions at 0 in each. So each record is read here, beside the -payload of its own call, into an entry that needs no payload to be read: the -row it is about becomes its key, and a related row its key and its values. - -The format is the endpoint's: ``gradients`` is ``[output][row][column]``, and -``columns`` are the model's preprocessed numerical columns, passed through. -""" +r"""Decode ``sdm_gradient_v1`` explanation records against their own call.""" from __future__ import annotations @@ -25,37 +14,23 @@ class GradientDecoder: - r"""Reads one call's explanations, beside that call's own payload. + r"""Decodes one call's explanation records into GradientExplanations. - Constructed once per call: with the payload for ``kumo-relational``, whose - explanations name rows of the related tables it sent, and with none for - ``kumo-tabular``, whose explanations are about the record's own row only. + Takes the call's payload for kumo-relational, to map related-row positions + back to keys and values; kumo-tabular needs none, its explanation is about + the record's own row. """ - #: The engine's own transport columns on a related table: the instance a - #: row was sampled for, and the key it numbers a keyless table's rows by - #: (`rfm/inference/payload.py`). Neither is a value the caller sent, so a - #: related row is given back without them. + #: Engine transport columns, not the caller's; stripped from related rows. INSTANCE_ID = 'instance_id' SYNTHETIC_KEY = '__node_id' def __init__(self, payload: Mapping[str, Any] | None = None) -> None: self._payload = payload - # How far each related row is from the entity, worked out once per - # call, and only if some record is explained. self._hops: dict[str, list[int | None]] | None = None def decode(self, items: Sequence[Any]) -> GradientExplanations: - r"""Each answered row's explanation, under the key naming that row. - - ``items`` have their ``id`` restored already, so the key is the one the - frame shows: the ``id``, or the ``row_index`` when the rows carry none. - A payload is given for a model that reads related tables. Then an entry - has ``related`` and no ``instance``: that model's ``instance_table`` is - the question row its engine sends, not the caller's, and the entity's - own table is among the related ones. Without it an entry has - ``instance`` and no ``related`` at all, rather than an empty one. - """ + r"""Each explained item's entry, keyed by ``id``, else ``row_index``.""" explained = [item for item in items if item.explanation] if not explained: return GradientExplanations() @@ -78,6 +53,7 @@ def decode(self, items: Sequence[Any]) -> GradientExplanations: ], } else: + # Its instance_table is the engine's question row, not the caller's. entry['related'], hops[key] = self._read_related( self._payload, details.get('related_tables') or {} ) @@ -91,17 +67,7 @@ def decode(self, items: Sequence[Any]) -> GradientExplanations: def _read_related( self, payload: Mapping[str, Any], blocks: Mapping[str, Any] ) -> tuple[dict[str, Any], dict[str, list[int | None]]]: - r"""Read a record's related blocks as rows of this call, not positions. - - The endpoint names each related row by its position in the table as - this call sent it, which the caller never saw and which a split request - restarts at 0 in every call. Each is given back as its primary key, - when the table has one of the caller's, and always as its values as - sent. - - How far each row is from the entity is worked out once per call, and - only if some record is explained. - """ + r"""A record's related blocks as keys and row values, with each row's hops.""" schema = payload.get('schema') or {} sent = (payload.get('predict') or {}).get('related_tables') or {} if self._hops is None: @@ -166,18 +132,7 @@ def _caller_key(cls, described: Mapping[str, Any]) -> str | None: def _hops_from_entity( cls, payload: Mapping[str, Any] ) -> dict[str, list[int | None]]: - r"""How many links each related row of this call is from its entity. - - Read from the relationships and key values sent, the same ones the - model joins on. A relationship with no ``source_table`` starts at the - instance table, and the rows it reaches are the entity's own: 0. Every - other one is followed either way, and a row is as far as its nearest - path. Joins stay within one instance, as the engine scopes them, so two - entities sharing a product never meet through it. - - Positions are this call's, one list per related table of its predict - half. ``None`` for a row no relationship reaches from an entity. - """ + r"""Links from the entity per related row of this call; ``None`` if unreached.""" schema = payload.get('schema') or {} predict = payload.get('predict') or {} instance = predict.get('instance_table') or {} @@ -236,12 +191,7 @@ def _hops_from_entity( def _joined_on( cls, table: Mapping[str, Any], columns: Sequence[str] ) -> list[Any]: - r"""What each row of ``table`` joins on through ``columns``. - - The instance a row belongs to, and its values in ``columns``; ``None`` - for a row holding a null there, which joins nothing. Empty when the - table does not carry every one of ``columns``. - """ + r"""Per row, ``(instance, values)`` in ``columns``; ``None`` on a null.""" names = list(table.get('columns') or []) if not columns or any(column not in names for column in columns): return [] From ff4ba96eb67f830cd3289d6c235723ec10b930e2 Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Thu, 29 Oct 2026 15:31:30 -0700 Subject: [PATCH 19/25] Say keys appear only for a related table's join-column key Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jana Gagacheva --- CHANGELOG.md | 5 ++-- docs/about/architecture.md | 7 +++--- docs/reference/prediction-output.md | 24 +++++++++---------- .../src/sdm_client/result.py | 5 ++-- skills/relational/SKILL.md | 2 +- .../references/beyond-one-answer.md | 8 +++---- 6 files changed, 27 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb08b2450..081119364 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,8 +35,9 @@ these packages. All five distributions share one version and one entry. request has no entity id. Each entry is a plain dict of `outputs` and, on `kumo-tabular`, the row's own columns under `instance`, or, on `kumo-relational`, every table, the entity's own included, under `related`, - whose rows carry their primary key when the table has one and always their - values as sent. It is kept out of + whose rows always carry their values as sent; `keys` is present when the + table's key column is one the model receives, i.e. a column a relationship + joins on; otherwise rows are identified by their values. It is kept out of `result.to_pandas()`; `result.explanations.to_pandas()` flattens it into one line per row, output and column, with each row's `hop` from the entity. diff --git a/docs/about/architecture.md b/docs/about/architecture.md index f17f558a4..7e3aa4845 100644 --- a/docs/about/architecture.md +++ b/docs/about/architecture.md @@ -594,9 +594,10 @@ position in the tables as submitted. The caller never sees those tables: the related ones are gathered by this client, and a keyless one is identified on the wire only by the engine's own synthetic node id. So the client turns each position back into what the caller can recognise. The predicted row's position -is dropped, since the mapping's key already names it. A related row carries its -primary key when its table has one, and always its values as sent, so a keyless -table's rows are still identified, by their contents. +is dropped, since the mapping's key already names it. A related row always +carries its values as sent; `keys` is present when the table's key column is +one the model receives, i.e. a column a relationship joins on; otherwise rows +are identified by their values. **Translated per call, before merging.** A request split across several calls numbers its positions from zero in each call. Read after merging, the second diff --git a/docs/reference/prediction-output.md b/docs/reference/prediction-output.md index dc36ae9fa..17bdf7b64 100644 --- a/docs/reference/prediction-output.md +++ b/docs/reference/prediction-output.md @@ -157,7 +157,7 @@ Each entry is a plain nested dict: 'related': { # kumo-relational only: every table, # the entity's own included '
': { - 'keys': [...], # the table's primary key per row; absent if the table has none + 'keys': [...], # each row's key; only when a relationship joins on the key column 'rows': [{...}, ...], # each row's values as sent to the model 'columns': [...], # model's preprocessed numerical columns 'gradients': [[[...], ...], ...], # [output][row][column], rows aligned with keys and rows @@ -203,9 +203,10 @@ by path. ### Related rows -A related table's rows carry `keys`, the table's primary key per row, when the -table has one, and always `rows`, each row's values as sent to the model. A -keyless table has no `keys`, so its rows are identified by their values alone. +A related table's rows always carry `rows`, each row's values as sent to the +model. `keys` is present when the table's key column is one the model receives, +i.e. a column a relationship joins on; otherwise rows are identified by their +values. Keyed or keyless changes only how a row is labelled; both get gradients in the same way. @@ -221,9 +222,9 @@ result.explanations['cust_17'] # 'gradients': [[0.01, -0.03], [-0.01, 0.03]]}} ``` -`kumo-relational`, user 5 with one order and two login events. `orders` has a -primary key (`order_id`), so its rows carry `keys`. `events` has none, so its -rows are identified by their values alone. Both are part of the same entry: +`kumo-relational`, user 5 with one order and two login events. `users` carries +`keys` because `user_id` is the join column; `orders` and `events` are +identified by their values. All are part of the same entry: ```python result.explanations['user_5'] @@ -234,8 +235,7 @@ result.explanations['user_5'] # 'rows': [{'user_id': 'user_5', 'tenure_days': 410}], # 'columns': ['tenure_days'], # 'gradients': [[[-0.05]], [[0.05]]]}, -# 'orders': {'keys': ['order_9003'], -# 'rows': [{'user_id': 'user_5', 'order_id': 'order_9003', 'amount': 12.0}], +# 'orders': {'rows': [{'user_id': 'user_5', 'order_id': 'order_9003', 'amount': 12.0}], # 'columns': ['amount'], # 'gradients': [[[-0.02]], [[0.02]]]}, # 'events': {'rows': [{'user_id': 'user_5', 'kind': 'login', 'at': '2026-10-01T09:00:00'}, @@ -253,7 +253,7 @@ across all entities: ``` id table hop key row output column gradient user_5 users 0 user_5 0 probability_yes tenure_days 0.05 -user_5 orders 1 order_9003 0 probability_yes amount 0.02 +user_5 orders 1 None 0 probability_yes amount 0.02 user_5 products 2 prod_77 0 probability_yes price 0.004 user_5 events 1 None 1 probability_yes kind_login -0.02 ``` @@ -264,8 +264,8 @@ user_5 events 1 None 1 probability_yes kind_login -0.02 - `hop` is how many links the row is from the entity, 0 for the entity's own table. The client works it out from the relationships and foreign-key values it sent. It is `None` for a row no relationship reaches. -- `key` is the row's primary key, or `None` for a keyless table. On a - `kumo-tabular` `instance` line it is the row's `id`, or `None` when the +- `key` is the row's entry in `keys`, or `None` when the table has no `keys`. + On a `kumo-tabular` `instance` line it is the row's `id`, or `None` when the request has no entity key. `row` is the row's position in that entity's `related[table]['rows']`, and 0 on a `kumo-tabular` `instance` line. - `output` names the score by the column of `result.to_pandas()` that holds diff --git a/packages/structured-data-models-client/src/sdm_client/result.py b/packages/structured-data-models-client/src/sdm_client/result.py index bb3406828..70296eed6 100644 --- a/packages/structured-data-models-client/src/sdm_client/result.py +++ b/packages/structured-data-models-client/src/sdm_client/result.py @@ -100,8 +100,9 @@ def to_pandas(self) -> pd.DataFrame: name for a related one, the ``kumo-relational`` entity's own table among them. ``hop`` is how many links the row is from the entity: 0 for the entity itself, and ``None`` where the rows sent do not link - it. ``key`` is the related row's primary key, ``None`` for a keyless - table, and the row's own ``id`` for its instance block. ``row`` is the + it. ``key`` is the related row's entry in ``keys``, present only when a + relationship joins on the table's key column, else ``None``; for an + instance block it is the row's own ``id``. ``row`` is the position in that entry's ``related[table]['rows']``. ``output`` names the score as the column of ``result.to_pandas()`` that holds it. """ diff --git a/skills/relational/SKILL.md b/skills/relational/SKILL.md index 2062ccfb0..69cb33328 100644 --- a/skills/relational/SKILL.md +++ b/skills/relational/SKILL.md @@ -199,7 +199,7 @@ and every related row gathered for it, each under its table's name in entry = result.explanations['user_5'] # the entity whose id is 'user_5' entry['related']['users'] # the entity's own row, hop 0 orders = entry['related']['orders'] # one block per related table -orders['keys'] # each row's primary key; absent if the table has none +entry['related']['users']['keys'] # present only when a relationship joins on the table's key column orders['rows'] # each row's values as sent orders['gradients'] # [output][row][column] result.explanations.to_pandas() # one line per (row, output, column), with its hop; the entity's users line is hop 0 diff --git a/skills/relational/references/beyond-one-answer.md b/skills/relational/references/beyond-one-answer.md index 7b2996b58..293987622 100644 --- a/skills/relational/references/beyond-one-answer.md +++ b/skills/relational/references/beyond-one-answer.md @@ -123,10 +123,10 @@ An entry holds `outputs`, the explained scores in order, and `related`, one block per table the client gathered for that entity. The entity's own row is one of those blocks, under its table's name, such as `related['users']`, at hop 0. -A block carries `keys`, each row's primary key, when the table has one, and -always `rows`, each row's values as sent, with `columns` and `gradients` as -`[output][row][column]`, aligned with those rows. A keyless table's rows are -identified by their values alone, and get gradients the same way. +A block always carries `rows`, each row's values as sent, with `columns` and +`gradients` as `[output][row][column]`, aligned with those rows. `keys` is +present when the table's key column is one the model receives, i.e. a column a +relationship joins on; otherwise rows are identified by their values. `columns` are the model's preprocessed numerical columns: categoricals and datetimes may appear as derived columns, and a table without numerical From cda1701639fd2c82088f4f9624e423eb2a7b0547 Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Thu, 29 Oct 2026 15:31:36 -0700 Subject: [PATCH 20/25] Expect keys only for join columns and explanation as served The kumo-relational NIM sends a related table's primary key only when a relationship joins on it, so orders blocks carry rows without keys, and it now serves explanation for every relational target. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jana Gagacheva --- .../tests/test_live_relational.py | 45 ++++++++++--------- tests/test_skills.py | 15 +++---- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/packages/structured-data-models-client/tests/test_live_relational.py b/packages/structured-data-models-client/tests/test_live_relational.py index e99959d4e..dc91bd17d 100644 --- a/packages/structured-data-models-client/tests/test_live_relational.py +++ b/packages/structured-data-models-client/tests/test_live_relational.py @@ -353,12 +353,15 @@ def test_run_mode_is_refused_as_an_option(query): EXPLAIN = ['prediction', 'probabilities', 'explanation'] BINARY_QUERY = 'PREDICT COUNT(orders.*, 0, 30, days) > 2 FOR EACH users.user_id' -#: Each keyed table of ``tables()`` and its primary key: its related blocks -#: are expected to carry ``keys``. -KEYS = {'users': 'user_id', 'orders': 'order_id'} -#: The tables of ``tables()`` with no primary key: their related blocks carry -#: ``rows`` and no ``keys``. -KEYLESS = {'events'} +#: The table whose related blocks carry ``keys``, and its key. The engine +#: sends a related table's primary key only when a relationship joins on it: +#: ``user_id`` is the join column, so only ``users`` is keyed. +KEYS = {'users': 'user_id'} +#: The tables whose related blocks carry ``rows`` and no ``keys``, each with a +#: column whose sent values the returned rows are checked against: ``orders``, +#: whose ``order_id`` no relationship joins on, and ``events``, which has no +#: primary key. +KEYLESS = {'orders': 'price', 'events': 'value'} @pytest.fixture(scope='module') @@ -373,9 +376,10 @@ def assert_explained_as_sent(result, asked): among the related ones. Each block's ``keys`` are the primary keys of its ``rows``, whose values are the caller's for that key and belong to that entity, and its gradients are ``[output][row][column]``, aligned with - ``rows``. A keyless table's block has ``rows`` and no ``keys``, each row - the caller's columns of one of that entity's rows. In ``to_pandas()`` the - entity's own table is at hop 0, the others at 1 or more, and a keyless + ``rows``. Keys come back only for a join column, so only ``users`` has + them; every other block has ``rows`` and no ``keys``, each row the + caller's columns of one of that entity's rows. In ``to_pandas()`` the + entity's own table is at hop 0, the others at 1 or more, and an unkeyed table's lines have no ``key``. """ sent_tables = tables() @@ -404,13 +408,14 @@ def assert_explained_as_sent(result, asked): for by_column in by_row: assert len(by_column) == len(block['columns']) if name in KEYLESS: - assert 'keys' not in block, f'{name} has no key and got keys' + assert 'keys' not in block, f'{name} is not joined on its key' own_rows = keyless[name][keyless[name]['user_id'] == entity] + column = KEYLESS[name] for row in block['rows']: # The caller's columns only: no instance_id, no __node_id. assert set(row) <= set(keyless[name].columns) assert int(row['user_id']) == entity - assert row['value'] in set(own_rows['value']) + assert row[column] in set(own_rows[column]) continue assert 'keys' in block, f'{name} has a key and came back without' assert len(block['keys']) == len(block['rows']) @@ -420,10 +425,7 @@ def assert_explained_as_sent(result, asked): assert int(row[KEYS[name]]) == int(row_key) assert int(row['user_id']) == entity sent = graph[name].loc[int(row_key)] - if name == 'orders': - assert row['price'] == sent['price'] - else: - assert row['segment'] == sent['segment'] + assert row['segment'] == sent['segment'] assert [int(one) for one in related['users']['keys']] == [entity] flat = result.explanations.to_pandas() @@ -461,8 +463,10 @@ def test_every_entity_is_explained_with_its_own_related_rows( Assumptions: - The NIM explains every predict row, both probabilities of a binary target. - - ``users`` and ``orders`` have a primary key and ``events`` has none, - so both kinds of related block are read back. + - Keys come back only for a join column: ``users`` (joined on + ``user_id``) carries ``keys``; ``orders`` (``order_id`` joins nothing) + and ``events`` (no primary key) carry ``rows`` and no ``keys``, so + both kinds of related block are read back. """ asked = [0, 1, 2, 3] request = binary_query.materialize(indices=asked) @@ -479,11 +483,12 @@ def test_a_request_sent_in_several_calls_explains_every_entity( ): """A request over the per-call cap is sent in calls that each number their rows from 0, and every entity still gets one entry, with its own - related rows and keys. + related rows, keyed only where the key is a join column. Assumptions: - - ``events`` has no primary key, so its rows are checked by entity and - value rather than by key. + - Keys come back only for a join column, so only ``users`` has them; + ``orders`` and ``events`` rows are checked by entity and value rather + than by key. - ``rows_per_call=2`` stands in for the cap, as in the batching test above, and splits eight entities into four calls. """ diff --git a/tests/test_skills.py b/tests/test_skills.py index db26b0669..79b5e4d95 100644 --- a/tests/test_skills.py +++ b/tests/test_skills.py @@ -2414,15 +2414,15 @@ def test_the_tabular_outputs_narrow_by_task_as_the_skill_says(): #: fails here rather than making the skill quietly wrong. SERVED = { 'COUNT(orders.*, 0, 30, days)': ( - ('prediction', 'quantiles'), + ('prediction', 'quantiles', 'explanation'), tuple(f'quantile_0.{one}' for one in range(1, 10)), ), 'COUNT(orders.*, 0, 30, days) > 2': ( - ('prediction', 'probabilities'), + ('prediction', 'probabilities', 'explanation'), ('probability_False', 'probability_True'), ), 'users.segment': ( - ('prediction', 'probabilities'), + ('prediction', 'probabilities', 'explanation'), ('probability_a', 'probability_b', 'probability_c'), ), } @@ -2478,7 +2478,8 @@ def test_the_relational_endpoint_serves_the_matrix_the_skill_gives( def test_the_relational_endpoint_explains_every_entity(target, schema): """What the skill shows of an explanation: one per entity asked about, under its id, with the entity's own table among its related ones and no - block for the engine's question row. If a deployment stops serving one, + block for the engine's question row. Keys come back only for a join + column, so only ``users`` is keyed; other blocks carry rows alone. If a deployment stops serving one, this fails and the skill is the thing to change.""" from sdm_client import KumoRelational, SDMClient @@ -2504,10 +2505,8 @@ def test_the_relational_endpoint_explains_every_entity(target, schema): assert own is not None, f'{target} gave {key} no block of its own' assert [int(one) for one in own['keys']] == [int(key)] for name, block in entry['related'].items(): - # Every table of `schema` has a primary key. - assert len(block.get('keys', ())) == len(block['rows']), ( - f'{name} has a key and came back without one per row' - ) + # Only `users` is joined on its key; other blocks need no keys. + assert block['rows'], f'{name} came back with no rows' assert len(block['gradients']) == len(entry['outputs']) for by_row in block['gradients']: assert len(by_row) == len(block['rows']), ( From 924842af24e228ce9131ed58e1cb016f22fd42ab Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Thu, 29 Oct 2026 15:41:47 -0700 Subject: [PATCH 21/25] Compare the explained tables against a set, not a dict Co-Authored-By: Claude Fable 5.1 Signed-off-by: Jana Gagacheva --- .../structured-data-models-client/tests/test_live_relational.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/structured-data-models-client/tests/test_live_relational.py b/packages/structured-data-models-client/tests/test_live_relational.py index dc91bd17d..039a3b9c4 100644 --- a/packages/structured-data-models-client/tests/test_live_relational.py +++ b/packages/structured-data-models-client/tests/test_live_relational.py @@ -400,7 +400,7 @@ def assert_explained_as_sent(result, asked): assert outputs == 2 related = entry['related'] assert 'users' in related, 'the entity has no block of its own' - assert set(related) <= set(KEYS) | KEYLESS + assert set(related) <= set(KEYS) | set(KEYLESS) for name, block in related.items(): assert len(block['gradients']) == outputs for by_row in block['gradients']: From 659b425956cbfe87abf0badf9636ab07d5c8eb82 Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Thu, 29 Oct 2026 15:43:36 -0700 Subject: [PATCH 22/25] remove the tracker file --- gradient_xai_plan.md | 188 ------------------------------------------- 1 file changed, 188 deletions(-) delete mode 100644 gradient_xai_plan.md diff --git a/gradient_xai_plan.md b/gradient_xai_plan.md deleted file mode 100644 index 59f45379e..000000000 --- a/gradient_xai_plan.md +++ /dev/null @@ -1,188 +0,0 @@ -# Gradient explanations (XAI) in the client: plan and tracker - -The shared tracker for Jana, Claude and subagents. Update a row's status when it changes, and -record who decided and when. Nothing is built from a row marked NEEDS DECISION. - -**Context** -- **Client PR #34:** branch `jg-enable-explain`. Rebased onto main at `12955d397` as - `a01c08d3a` + `78b0a5ebb`, and pushed; the PR head is `78b0a5ebb`. The explanation work is - local commits on top, `793a2a8c4`..`9de1214f8`, not pushed. -- **NIM:** `structured-data-nims` origin/main `0a8230d` explains every predict row and returns one - `explanation` per prediction record. -- **Wire format** `sdm_gradient_v1`: described in the NIM repo at - `sdm_profiles/kumo_{relational,tabular}_sdm/USER_GUIDE.md`, section "Gradient Explanations". - `gradients` is `[output][row][column]`. `row_indices` are positions in the tables as submitted, - not keys. - -**Status values:** `NEEDS DECISION` (Jana chooses; the end-state column holds a proposal) · -`TO DO` (agreed, not built yet; who agreed and when, in brackets) · `WAITING (on X)` (blocked by -another row or something outside) · `DONE` (finished, with where) - -| Issue | End state / fix | Status | -|---|---|---| -| **R1** PR #34 is behind main and conflicts in `tabular.py`, `conftest.py` and `CHANGELOG.md` | Rebased onto main, conflicts resolved, the PR stays one commit; CI-equivalent checks pass locally | DONE: `a01c08d3a`; 2188 passed / 379 skipped (missing extras) / 0 failed | -| **R2** Docs said kumo-tabular doesn't explain; a test docstring stated the old `run_mode='fast'` rule as fact | Architecture capability table shows ✅; the docstring is scoped to "a deployment that…" | DONE: `78b0a5ebb` | -| **R3** The rebased branch is only local | Pushed; PR #34 shows the new commits | DONE: pushed 2026-10-28 18:59 PT, PR head `78b0a5ebb` | -| **N1** The NIM refused explanations over more than one row | The NIM explains every predict row | DONE: Jana, NIM `0a8230d` | -| **N2** Concern: a related row shared by two entities would get a mixed gradient credited to only one of them | No shared rows reach the NIM | DONE: verified by code reading plus a library test. The engine sampler copies rows per predict row, and the NIM scopes joins by `instance_id`. Not run end to end. | -| **L1** `explanation_predict_rows` in `TABULAR_LIMITS` and `RELATIONAL_LIMITS` (`models.py`) | Removed | DONE: `0da19289e` (`models.py`) | -| **L2** Tabular `_check_explanation_rows` (added by #34) | Removed, with its calls | DONE: `0da19289e` (`adapters/tabular.py`) | -| **L3** Relational one-entity explanation check (`adapters/relational.py` ~:380-400) | Removed | DONE: `0da19289e` (`adapters/relational.py`) | -| **L4** Other one-row assumptions may hide in the materializer, `decompose.py` or `client.py` | Grepped; none remain | DONE: `0da19289e`; grepped client, pql and engine, none left | -| **A1** `result.explanation` keeps only the first record's explanation and silently drops the rest | Removed. Replaced by `result.explanations`, a mapping from row key to that row's explanation, kept out of the DataFrame | DONE: `23db8a188` (`result.py` `explanations`) | -| **A2** Key for each explanation | Whatever identifies the row in `to_pandas()`: `id`; for forecasts `(id, forecast_step)`; `row_index` when the request has no entity key. The same rule for both models | DONE: `23db8a188` (`wire/explanations.py`, `client.py` `_by_step`, `_numbered_from`) | -| **A3** The instance block's `row_indices` is a position, not a key | Dropped, since the mapping key names the row; gradients become `[output][column]` | DONE: `23db8a188` | -| **A4** Related-table `row_indices` are positions in tables the caller never sees; keyless tables only carry the engine's synthetic `__node_id` (`payload.py:587-592`) | Each related row carries its primary key when the table has one, and always the row's values as sent; gradients stay `[output][row][column]`, aligned with those rows | DONE: `23db8a188` (`adapters/relational.py` `_related_rows_of`) | -| **A5** When a request is split across calls (#36), positions restart at 0 in each call | Translated inside each call's `answered()`, before merging | DONE: `23db8a188` (translated in each call's `answered()`) | -| **A7** Forecast `{'by_forecast_step': ...}` aggregation (`client.py` ~:1241-1305) | Removed; per-row keys `(id, forecast_step)` replace it | DONE: `23db8a188` (`client.py`) | -| **A9** `result.explanations` when explanations weren't requested | An empty dict | DONE: `23db8a188` | -| **A11** Type of an explanation entry | Plain dicts (A15). `result.explanations.to_pandas()` adds one flat table, with `hop` (A17). See "Reading it" in Proposed API | DONE: `23db8a188` (`Explanations.to_pandas()`) | -| **A12** `outputs` and `columns` repeat in every record | Kept per entry, as the NIM returns them | DONE: `23db8a188` | -| **A13** `columns` are the model's preprocessed columns; categoricals and datetimes appear as derived columns | Passed through as the NIM returns them, and documented (D3) | DONE: passed through; documented in `bff621aba` | -| **A14** `__contains__`, has-field and repr refer to `explanation` | Updated for `explanations` | DONE: `23db8a188` | -| **A15** Shape of one entry | `{'outputs': [...], 'instance': {'columns', 'gradients' [output][column]}, 'related': {table: {'keys', 'rows' (values as sent), 'columns', 'gradients' [output][row][column]}}}` | DONE: `23db8a188`, `ca99c782f` | -| **A16** Each model's entry has only the tables the caller sent. kumo-tabular has no related tables. For kumo-relational the client's engine sends the entity as two tables: a question row (`instance_table`: `instance_id`, entity reference, anchor time; no caller columns) and the entity's own table inside `related_tables`, and the NIM explains both | kumo-tabular entries have no `related` key. kumo-relational entries have no `instance` key: the client drops the question row's block when it reads the reply, and the entity's own table appears under `related` like every other table, at hop 0 | DONE: `ca99c782f` (relational entries have no `instance`) | -| **A17** The `hop` column in `to_pandas()` (A11) needs the client to work out each related row's distance from the entity | Computed from the relationships and foreign-key values in the payload sent, per call; checked against a real multi-hop payload | DONE: `23db8a188` (`_hops_from_the_entity`); checked on the golden one-hop payload; deeper hops are checked by T4 | -| **B2** The `live_nim` parity case in `test_every_backend_answers_alike.py` expects a NIM to refuse relational explanations | The case is removed; T1 covers both behaviours (the in-process backend refuses, the NIM answers) | DONE: `b2a30cd3e` | -| **T1** Tests for the new behaviour | Written per the `unittesting` skill. Cover tabular multi-row; relational multi-row with key and value translation; keyless related table; a split request whose second call restarts at 0; forecast keys; the in-process backend refusing `'explanation'`; `to_pandas()` | DONE: `a79c00aa0`, `ca99c782f` (unit tests with mocked NIM replies) | -| **T2** Old tests assert the one-row refusals and `result.explanation` | Updated or removed | DONE: `0da19289e`, `23db8a188`, `36de54755` | -| **T3** Local checks | ruff, format, mypy, hygiene and the CI pytest command all pass; counts recorded here | DONE: ruff, format, mypy (184 files), hygiene, client suite 1580 passed / 256 skipped / 0 failed, root tests 174 passed / 109 skipped (2026-10-29, rerun by the main agent) | -| **T4** The client has never run against a NIM that has `0a8230d` | Explanation cases (tabular and relational, multi-row, keyless related table, a split request) added to `test_live_nim.py` and `test_live_relational.py`, run once against a NIM built from `0a8230d`; output checked against A1-A5 | WAITING (on the rebuilt NIM image; Jana names the endpoint). Test code written: `9de1214f8` (`test_live_nim.py`, `test_live_relational.py`, `tests/test_skills.py`). Keyless related table added to the live fixture: `716f029cf` (agreed: Jana, 2026-10-29). Also at T4 time: add `explanation` to `SERVED` in `tests/test_skills.py` so the skill-matrix live test stops expecting a refusal (agreed: Jana, 2026-10-29). Both images triggered in the NIM repo CI (`SDM_PROFILE=both`) 2026-10-29 | -| **D1** `result.py` module docstring shows `result.explanation` | Shows `result.explanations` and its keys | DONE: `23db8a188` | -| **D2** `docs/reference/limits.md` explanation rows | The row limit is removed | DONE: `793a2a8c4` | -| **D3** `docs/reference/prediction-output.md` | Has a section on explanations: shape, keys, related rows, preprocessed columns | DONE: `793a2a8c4`, `964ded993`, `bff621aba` | -| **D4** `docs/about/architecture.md` | Explains the explanation design and why (from this file) | DONE: `793a2a8c4`, `bff621aba` | -| **D5** `skills/tabular/SKILL.md` and `skills/relational/SKILL.md` say one row per explanation | Multi-row, `result.explanations`, with an example | DONE: `793a2a8c4`, `bff621aba` | -| **D6** `qa/probe_tabular.py` | Probes a multi-row explanation | DONE: `793a2a8c4`; `qa/probe_relational.py` too (`964ded993`, `bff621aba`); not run, no NIM | -| **V1** CHANGELOG `## Unreleased` | Changed: multi-row explanations. Removed: `result.explanation` and `by_forecast_step`. Added: `result.explanations` | DONE: `793a2a8c4`, `bff621aba` | - -## Proposed API - -The IDs in brackets refer to rows in the tracker table above. Each item's status lives only -there. - -### Asking - -Asking is unchanged: add `'explanation'` to `outputs`. There is no row limit (L1-L3). - -```python -result = client.predict(request, model=KumoRelational(), - outputs=['prediction', 'probabilities', 'explanation']) -``` - -### `result.explanations` - -`result.explanations` is a mapping from a row's key to that row's explanation. It is kept out of -`result.to_pandas()` (A1). `result.explanation` and `{'by_forecast_step': ...}` no longer exist -(A1, A7). - -| Request | Key | Example | -|---|---|---| -| has an entity id | the row's `id`, the same value as in `to_pandas()['id']` | `result.explanations['user_5']` | -| forecast | `(id, forecast_step)` | `result.explanations[('user_5', 3)]` | -| no entity id (single-table tabular) | `row_index` (A2) | `result.explanations[0]` | - -### One entry (A15) - -```python -{ - 'outputs': [...], # the explained scores, in order (see below) - 'instance': { - 'columns': [...], # model's preprocessed numerical columns (A13) - 'gradients': [[...], ...], # [output][column] - }, - 'related': { # absent for kumo-tabular; for kumo-relational it - # holds every table incl. the entity's own, and - # there is no 'instance' (A16) - '
': { - 'keys': [...], # the table's primary key per row; absent if the table has none - 'rows': [{...}, ...], # each row's values as sent to the model - 'columns': [...], # model's preprocessed numerical columns (A13) - 'gradients': [[[...], ...], ...], # [output][row][column], rows aligned with keys and rows - }, - }, -} -``` - -`outputs` is copied from the NIM: -- classification gives every class probability in declared order, e.g. - `{'kind': 'probability', 'class': 'yes'}`; -- regression gives the returned prediction, `{'kind': 'prediction', 'statistic': 'mean'}` - (or `'median'`), plus each returned quantile when `quantiles` is requested. - -A gradient is the change in that output score per unit change in that preprocessed column. It is -a diagnostic, not a causal effect or a feature-importance ranking. - -### Which tables get gradients - -| Model | Tables with gradients | -|---|---| -| kumo-tabular | the one table: each predicted row's own columns | -| kumo-relational | the predicted entity's own table, plus every related table in the subgraph the client gathered for that entity (all tables reached within the hop limit, up to 6). The engine's question row (`instance_table`) is explained by the NIM too, but the client drops that block (A16) | - -- Only the rows being predicted and their related rows get gradients. The context rows (the - labelled examples the model compares against) get none (`EnableInputGradients` in - `sdm/explain/gradient.py`). -- Only numerical inputs as the model sees them get gradients. Categoricals and datetimes appear as - derived numeric columns; a table with none has an empty `columns` (A13). -- Gradients are per row, not per path. A row reached through several paths, for example a product - bought in two of the user's orders, appears once. Its gradient is its total effect through all - of them, and the model gives no split by path. - -### Examples (values illustrative) - -kumo-tabular, request with an id column: - -```python -result.explanations['cust_17'] -# {'outputs': [{'kind': 'probability', 'class': 'no'}, -# {'kind': 'probability', 'class': 'yes'}], -# 'instance': {'columns': ['age', 'balance'], -# 'gradients': [[0.01, -0.03], [-0.01, 0.03]]}} -``` - -kumo-relational, user 5 with one order and two login events. `orders` has a primary key -(`order_id`), so its rows carry `keys`. `events` has none, so its rows are identified by their -values alone. Both are part of the same entry: - -```python -result.explanations['user_5'] -# {'outputs': [{'kind': 'probability', 'class': 'no'}, -# {'kind': 'probability', 'class': 'yes'}], -# 'related': { -# 'users': {'keys': ['user_5'], -# 'rows': [{'user_id': 'user_5', 'tenure_days': 410}], -# 'columns': ['tenure_days'], -# 'gradients': [[[-0.05]], [[0.05]]]}, -# 'orders': {'keys': ['order_9003'], -# 'rows': [{'user_id': 'user_5', 'order_id': 'order_9003', 'amount': 12.0}], -# 'columns': ['amount'], -# 'gradients': [[[-0.02]], [[0.02]]]}, -# 'events': {'rows': [{'user_id': 'user_5', 'kind': 'login', 'at': '2026-10-01T09:00:00'}, -# {'user_id': 'user_5', 'kind': 'login', 'at': '2026-10-03T18:20:00'}], -# 'columns': ['kind_login', 'at_dayofweek'], # model-derived names, illustrative (A13) -# 'gradients': [[[0.01, 0.00], [0.02, -0.01]], -# [[-0.01, 0.00], [-0.02, 0.01]]]}}} -``` - -Keyed or keyless changes only how a row is labelled in the output. Both tables get gradients in -the same way. - -### Reading it (A11) - -Each entry is the plain nested dict shown above. `result.explanations.to_pandas()` gives one line -per (row, output, column), across all entities: - -``` -id table hop key row output column gradient -user_5 users 0 user_5 0 probability_yes tenure_days 0.05 -user_5 orders 1 order_9003 0 probability_yes amount 0.02 -user_5 products 2 prod_77 0 probability_yes price 0.004 -user_5 events 1 None 1 probability_yes kind_login -0.02 -``` - -- `hop` is how many links the row is from the entity (0 is the entity's own table; for kumo-tabular the one table, shown as `instance`). `None` for a row no relationship reaches. The client works - it out from the relationships and foreign-key values it sent (A17). -- `output` is the matching `result.to_pandas()` column name (`probability_yes`, `prediction`, `quantile_0.1`). -- `key` is the row's primary key, or `None` for a keyless table. `row` is the row's position in - that entity's `related[table]['rows']`. From 236349c7c98e5f1adfcb2e14e07336fbaf591215 Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Thu, 29 Oct 2026 16:46:50 -0700 Subject: [PATCH 23/25] Keep result.explanation and explanation= as deprecated names Both were public in 1.0.0. result.explanation returns the first record's raw explanation, or None, and PredictionResult(explanation=...) is accepted again; each raises DeprecationWarning naming result.explanations and removal in 2.0.0. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jana Gagacheva --- CHANGELOG.md | 14 ++++--- docs/reference/prediction-output.md | 4 +- .../src/sdm_client/result.py | 26 +++++++++++++ .../tests/test_envelope_and_result.py | 39 +++++++++++++++++++ tests/golden/public_api.json | 3 +- 5 files changed, 78 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 081119364..225a8a48e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,11 +60,15 @@ these packages. All five distributions share one version and one entry. explanation over more than one row and a `kumo-relational` one over more than one entity are no longer refused before sending. -### Removed - -- `result.explanation`, which held the first record's explanation only, and - the `{'by_forecast_step': ...}` it held for a forecast. Read - `result.explanations` instead. +### Deprecated + +- `result.explanation` and `PredictionResult(explanation=...)` are deprecated + in favour of `result.explanations` and `explanations=`, warn with + `DeprecationWarning`, and will be removed in 2.0.0. To upgrade, read a row's + explanation as `result.explanations[key]`, under the key naming that row in + `result.to_pandas()`. `result.explanation` returns the first record's raw + explanation only; the `{'by_forecast_step': ...}` it held for a forecast is + gone. ### Fixed diff --git a/docs/reference/prediction-output.md b/docs/reference/prediction-output.md index 17bdf7b64..fa73594bd 100644 --- a/docs/reference/prediction-output.md +++ b/docs/reference/prediction-output.md @@ -131,8 +131,8 @@ result = client.predict(request, model=KumoRelational(), ### Keys `result.explanations` maps a row's key to that row's explanation. It is kept -out of `result.to_pandas()`. There is no `result.explanation` and no -`{'by_forecast_step': ...}`. +out of `result.to_pandas()`. `result.explanation` is deprecated, see +`result.explanations`; there is no `{'by_forecast_step': ...}`. The key is whatever identifies the row in `to_pandas()`, by the same rule on both models: diff --git a/packages/structured-data-models-client/src/sdm_client/result.py b/packages/structured-data-models-client/src/sdm_client/result.py index 70296eed6..d51a89c76 100644 --- a/packages/structured-data-models-client/src/sdm_client/result.py +++ b/packages/structured-data-models-client/src/sdm_client/result.py @@ -36,6 +36,7 @@ class as columns rather than a row per class, while forecasting gives one row from __future__ import annotations +import warnings from collections.abc import Iterator, Mapping, Sequence from typing import Any @@ -242,7 +243,16 @@ def __init__( explanations: Mapping[Any, dict[str, Any]] | None = None, outputs: Sequence[str] = (), unscored_candidates: Sequence[Any] = (), + explanation: dict[str, Any] | None = None, ) -> None: + if explanation is not None: + warnings.warn( + 'PredictionResult(explanation=...) is deprecated; pass ' + 'explanations=. It will be removed in 2.0.0', + DeprecationWarning, + stacklevel=2, + ) + self._explanation = explanation self.model = model self.request_id = request_id self.items = tuple(items) @@ -313,6 +323,22 @@ def explanations(self) -> GradientExplanations: """ return self._explanations + @property + def explanation(self) -> dict[str, Any] | None: + r"""Deprecated: the first record's raw explanation; use ``explanations``.""" + warnings.warn( + 'result.explanation is deprecated; use result.explanations. ' + 'It will be removed in 2.0.0', + DeprecationWarning, + stacklevel=2, + ) + if self._explanation is not None: + return self._explanation + return next( + (item.explanation for item in self.items if item.explanation), + None, + ) + @property def unscored_candidates(self) -> tuple[Any, ...]: r"""The candidates a ranking left out, in the order they were named. diff --git a/packages/structured-data-models-client/tests/test_envelope_and_result.py b/packages/structured-data-models-client/tests/test_envelope_and_result.py index c2e5dd99d..d8db97f24 100644 --- a/packages/structured-data-models-client/tests/test_envelope_and_result.py +++ b/packages/structured-data-models-client/tests/test_envelope_and_result.py @@ -248,6 +248,45 @@ def test_an_explanation_is_absent_rather_than_a_different_return_type(): assert isinstance(result.to_pandas(), pd.DataFrame) +_RAW = {'format': 'sdm_gradient_v1', 'details': {'outputs': []}} + + +def test_the_deprecated_explanation_is_the_first_records_raw_one(): + """result.explanation warns and returns the first explained record's raw dict.""" + result = result_with( + {'row_index': 0, 'prediction': 1}, + {'row_index': 1, 'prediction': 0, 'explanation': _RAW}, + {'row_index': 2, 'prediction': 1, 'explanation': {'format': 'x'}}, + ) + with pytest.warns( + DeprecationWarning, match=r'use result\.explanations.*2\.0\.0' + ): + assert result.explanation == _RAW + + +def test_the_deprecated_explanation_is_none_and_still_warns_when_none_came(): + """result.explanation warns and returns None when no record is explained.""" + result = result_with({'row_index': 0, 'prediction': 1}) + with pytest.warns( + DeprecationWarning, match=r'result\.explanation is deprecated' + ): + assert result.explanation is None + + +def test_the_deprecated_explanation_keyword_is_accepted_and_read_back(): + """explanation= warns, is what result.explanation returns, and leaves explanations alone.""" + with pytest.warns(DeprecationWarning, match=r'pass explanations=.*2\.0\.0'): + result = PredictionResult( + model='kumo-tabular', + request_id='r-1', + items=[PredictionItem.from_dict({'row_index': 0, 'prediction': 1})], + explanation=_RAW, + ) + with pytest.warns(DeprecationWarning): + assert result.explanation == _RAW + assert result.explanations == {} + + def test_quantiles_are_spread_under_a_readable_prefix(): frame = result_with( {'row_index': 0, 'prediction': 2.0, 'quantiles': {'0.5': 2.0}} diff --git a/tests/golden/public_api.json b/tests/golden/public_api.json index 8b09df16c..1a46605e0 100644 --- a/tests/golden/public_api.json +++ b/tests/golden/public_api.json @@ -238,6 +238,7 @@ "bases": [], "kind": "class", "public_attributes": [ + "explanation", "explanations", "unscored_candidates" ], @@ -246,7 +247,7 @@ "merge": "(parts: 'Sequence[PredictionResult]') -> 'PredictionResult'", "to_pandas": "(self) -> 'pd.DataFrame'" }, - "signature": "(*, model: 'str', request_id: 'str', items: 'Sequence[PredictionItem]', metadata: 'dict[str, Any] | None' = None, explanations: 'Mapping[Any, dict[str, Any]] | None' = None, outputs: 'Sequence[str]' = (), unscored_candidates: 'Sequence[Any]' = ()) -> 'None'" + "signature": "(*, model: 'str', request_id: 'str', items: 'Sequence[PredictionItem]', metadata: 'dict[str, Any] | None' = None, explanations: 'Mapping[Any, dict[str, Any]] | None' = None, outputs: 'Sequence[str]' = (), unscored_candidates: 'Sequence[Any]' = (), explanation: 'dict[str, Any] | None' = None) -> 'None'" }, "PredictiveQuery": { "bases": [], From 21f9876d5a29e4f51464e0b29b5e4279c2c93f67 Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Thu, 29 Oct 2026 16:50:32 -0700 Subject: [PATCH 24/25] resolve comments --- CHANGELOG.md | 5 +- docs/reference/prediction-output.md | 6 +-- .../src/sdm_client/result.py | 49 ++++++++++++++++--- .../src/sdm_client/wire/explanations.py | 11 ++++- .../tests/test_relational_explanations.py | 24 +++++++++ .../tests/test_tabular_explanations.py | 25 ++++++++++ 6 files changed, 106 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 225a8a48e..d20079c09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,7 +58,10 @@ these packages. All five distributions share one version and one entry. Relational SDM NIM explains under either budget. - An explanation covers every scored row, on both models. A `kumo-tabular` explanation over more than one row and a `kumo-relational` one over more - than one entity are no longer refused before sending. + than one entity are no longer refused before sending. This needs a NIM built + from structured-data-nims `0a8230d` or later; the shipping releases + (Relational 1.1.0, Tabular 1.0.0) refuse `explanation`. The first release + that serves it will be named here when it is cut. ### Deprecated diff --git a/docs/reference/prediction-output.md b/docs/reference/prediction-output.md index fa73594bd..63890338b 100644 --- a/docs/reference/prediction-output.md +++ b/docs/reference/prediction-output.md @@ -139,9 +139,9 @@ both models: | Request | Key | Example | |---|---|---| -| has an entity id | the row's `id`, the same value as in `to_pandas()['id']` | `result.explanations['user_5']` | -| forecast | `(id, forecast_step)` | `result.explanations[('user_5', 3)]` | -| no entity id (single-table tabular) | `row_index` | `result.explanations[0]` | +| has an entity id, none repeated | the row's `id`, the same value as in `to_pandas()['id']` | `result.explanations['user_5']` | +| forecast | `(id, forecast_step)`, or `(row_index, forecast_step)` when ids repeat | `result.explanations[('user_5', 3)]` | +| no id column, or its ids repeat | `row_index` | `result.explanations[0]` | ### One entry diff --git a/packages/structured-data-models-client/src/sdm_client/result.py b/packages/structured-data-models-client/src/sdm_client/result.py index d51a89c76..2fcf1f63e 100644 --- a/packages/structured-data-models-client/src/sdm_client/result.py +++ b/packages/structured-data-models-client/src/sdm_client/result.py @@ -92,6 +92,8 @@ def __init__( # table, aligned with that table's `rows`. Kept beside the entries # rather than in them, because an entry is the endpoint's shape. self._hops = {key: dict(tables) for key, tables in (hops or {}).items()} + # Each entry's key as it would be under `row_index`, to re-key by. + self._rows: dict[Any, Any] = {} def to_pandas(self) -> pd.DataFrame: r"""Every gradient, one line each, across all explained rows. @@ -163,16 +165,28 @@ def to_pandas(self) -> pd.DataFrame: ).astype({'hop': 'Int64', 'row': 'int64', 'gradient': 'float64'}) def _renumbered(self, start: int) -> GradientExplanations: - r"""These, with each ``row_index`` key ``start`` places further on. + r"""These, with each ``row_index`` ``start`` places further on. A key that is an ``id`` is not a position and is left alone. """ - if not start or self._keyed_by != ('row_index',): + if not start: return self + moved = self._keyed_by == ('row_index',) return GradientExplanations( - {key + start: entry for key, entry in self.items()}, + { + key + start if moved else key: entry + for key, entry in self.items() + }, keyed_by=self._keyed_by, - hops={key + start: hops for key, hops in self._hops.items()}, + hops={ + key + start if moved else key: hops + for key, hops in self._hops.items() + }, + )._with_rows( + { + key + start if moved else key: row + start + for key, row in self._rows.items() + } ) def _at_step(self, step: Any) -> GradientExplanations: @@ -181,23 +195,42 @@ def _at_step(self, step: Any) -> GradientExplanations: {(key, step): entry for key, entry in self.items()}, keyed_by=(*self._keyed_by, 'forecast_step'), hops={(key, step): hops for key, hops in self._hops.items()}, + )._with_rows( + {(key, step): (row, step) for key, row in self._rows.items()} ) + def _with_rows(self, rows: Mapping[Any, Any]) -> GradientExplanations: + r"""These, recording each key's ``row_index`` key.""" + self._rows = dict(rows) + return self + + def _by_row(self) -> GradientExplanations: + r"""These, keyed by ``row_index`` in place of ``id``.""" + if self._keyed_by[0] == 'row_index': + return self + return GradientExplanations( + {self._rows[key]: entry for key, entry in self.items()}, + keyed_by=('row_index', *self._keyed_by[1:]), + hops={self._rows[key]: hops for key, hops in self._hops.items()}, + )._with_rows({row: row for row in self._rows.values()}) + @classmethod def _merged( cls, parts: Sequence[GradientExplanations] ) -> GradientExplanations: - r"""The explanations of several calls, as one. - - Each call answers about its own rows, so no key is in two parts. - """ + r"""The explanations of several calls, as one; by ``row_index`` if ids repeat.""" explained = [part for part in parts if part] if not explained: return cls() + keys = [key for part in explained for key in part] + mixed = len({part._keyed_by for part in explained}) > 1 + if mixed or len(set(keys)) < len(keys): + explained = [part._by_row() for part in explained] merged = cls(keyed_by=explained[0]._keyed_by) for part in explained: merged.update(part) merged._hops.update(part._hops) + merged._rows.update(part._rows) return merged diff --git a/packages/structured-data-models-client/src/sdm_client/wire/explanations.py b/packages/structured-data-models-client/src/sdm_client/wire/explanations.py index 9c7231b9e..1eabb8e45 100644 --- a/packages/structured-data-models-client/src/sdm_client/wire/explanations.py +++ b/packages/structured-data-models-client/src/sdm_client/wire/explanations.py @@ -30,11 +30,13 @@ def __init__(self, payload: Mapping[str, Any] | None = None) -> None: self._hops: dict[str, list[int | None]] | None = None def decode(self, items: Sequence[Any]) -> GradientExplanations: - r"""Each explained item's entry, keyed by ``id``, else ``row_index``.""" + r"""Each explained item's entry, by ``id`` if those are unique, else ``row_index``.""" explained = [item for item in items if item.explanation] if not explained: return GradientExplanations() - by_id = any(item.id is not None for item in explained) + ids = [item.id for item in explained] + unique = len(set(ids)) == len(ids) + by_id = unique and any(key is not None for key in ids) entries: dict[Any, dict[str, Any]] = {} hops: dict[Any, dict[str, list[int | None]]] = {} for item in explained: @@ -62,6 +64,11 @@ def decode(self, items: Sequence[Any]) -> GradientExplanations: entries, keyed_by=('id',) if by_id else ('row_index',), hops=hops, + )._with_rows( + { + item.id if by_id else item.row_index: item.row_index + for item in explained + } ) def _read_related( diff --git a/packages/structured-data-models-client/tests/test_relational_explanations.py b/packages/structured-data-models-client/tests/test_relational_explanations.py index f5ef6dc09..52f4bf523 100644 --- a/packages/structured-data-models-client/tests/test_relational_explanations.py +++ b/packages/structured-data-models-client/tests/test_relational_explanations.py @@ -447,6 +447,30 @@ def test_a_split_request_reads_each_call_against_its_own_rows( assert result.explanations[5]['related']['orders']['keys'] == [9003, 9004] +def test_a_split_request_whose_calls_repeat_an_id_keys_by_row_index( + gradient_explanation, +): + """When two calls each explain the same id, both entries are kept, each + under its row_index across the whole request. + + Assumptions: + - The request is gathered as two calls, each scoring user 5, so each + call's ids are unique but repeat across the calls. + - The client numbers the second call's row after the first call's one. + """ + client, _ = client_answering(gradient_explanation) + + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + result = client.predict( + split((5,), (5,)), model=KumoRelational(), outputs=list(EXPLAIN) + ) + + assert sorted(result.explanations) == [0, 1] + assert result.explanations[1]['related']['users']['keys'] == [5] + assert list(result.explanations.to_pandas().columns)[0] == 'row_index' + + def test_a_split_request_keeps_row_index_keys_apart(gradient_explanation): """Explanations kept under ``row_index`` move with their rows when a later call's rows are numbered after the earlier ones. diff --git a/packages/structured-data-models-client/tests/test_tabular_explanations.py b/packages/structured-data-models-client/tests/test_tabular_explanations.py index b8d0fcd08..d6218466e 100644 --- a/packages/structured-data-models-client/tests/test_tabular_explanations.py +++ b/packages/structured-data-models-client/tests/test_tabular_explanations.py @@ -13,6 +13,7 @@ import pandas as pd import pytest +from sdm_client import Request from sdm_client.adapters.tabular import TabularAdapter from sdm_client.base import check_request_against_model from sdm_client.models import KUMO_TABULAR @@ -207,6 +208,30 @@ def test_rows_without_a_key_are_explained_under_their_row_index( assert result.explanations[2]['instance']['gradients'] == [[-3.0], [3.0]] +def test_rows_whose_ids_repeat_are_explained_under_their_row_index( + tabular_request, gradient_explanation +): + """Two rows sharing an id are both explained, each under its row_index. + + Assumptions: + - The request's two rows both carry ``row_id`` 90. + - The NIM answers each row under ``'[90]'`` with its own explanation. + """ + built = tabular_request(rows=2) + request = Request( + context=built.context.instance_table, + predict=built.predict.instance_table.assign(row_id=[90, 90]), + target='churn', + task='classification', + ) + + result = answered(request, gradient_explanation) + + assert sorted(result.explanations) == [0, 1] + assert result.explanations[1]['instance']['gradients'] == [[-2.0], [2.0]] + assert list(result.explanations.to_pandas().columns)[0] == 'row_index' + + def test_a_tabular_entry_has_no_related_tables( tabular_request, gradient_explanation ): From 04353de461a40e632924916915746d405b867386 Mon Sep 17 00:00:00 2001 From: Jana Gagacheva Date: Thu, 29 Oct 2026 16:53:13 -0700 Subject: [PATCH 25/25] Name no internal repository in the changelog Co-Authored-By: Claude Fable 5.1 Signed-off-by: Jana Gagacheva --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d20079c09..0c16a8877 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,10 +58,10 @@ these packages. All five distributions share one version and one entry. Relational SDM NIM explains under either budget. - An explanation covers every scored row, on both models. A `kumo-tabular` explanation over more than one row and a `kumo-relational` one over more - than one entity are no longer refused before sending. This needs a NIM built - from structured-data-nims `0a8230d` or later; the shipping releases - (Relational 1.1.0, Tabular 1.0.0) refuse `explanation`. The first release - that serves it will be named here when it is cut. + than one entity are no longer refused before sending. This needs a NIM newer + than the shipping releases (Relational 1.1.0, Tabular 1.0.0), which refuse + `explanation`. The first release that serves it will be named here when it + is cut. ### Deprecated