feat(sdk): expose decode path and kind on JSON decode failures

Cargo.toml

@@ -22,6 +22,7 @@ pyo3-stub-gen = { version = "0.23", default-features = false, features = ["infer
 reqwest = { version = "0.13.4", default-features = false }
 serde = { version = "1.0.229", features = ["derive"] }
 serde_json = { version = "1.0.149", features = ["arbitrary_precision"] }
+serde_path_to_error = "0.1.20"
 syn = { version = "2.0", features = ["full", "extra-traits"] }
 thiserror = "2.0"
 tokio = { version = "1.52.3", features = ["io-util", "macros", "net", "rt-multi-thread", "time"] }

crates/fmp-py/README.md

@@ -142,12 +142,12 @@ package root):
 
 | Exception | Raised when |
 |-----------|-------------|
-| `FmpError` | base class; carries `category`, `endpoint`, `status`, `body`, `body_truncated` |
+| `FmpError` | base class; carries `category`, `endpoint`, `status`, `body`, `body_truncated`, `decode_path`, `decode_kind` |
 | `FmpConfigError` | the client cannot be built (missing key, bad URL, insecure auth) |
 | `FmpValidationError` | an argument is rejected before the request |
 | `FmpTransportError` | the request never produced a response |
 | `FmpStatusError` | the provider answered with a non-success status |
-| `FmpDecodeError` | the body could not be decoded into the documented shape |
+| `FmpDecodeError` | the body could not be decoded into the documented shape; `decode_path` names the member (for example `[37].beta`) and `decode_kind` the reason, never the value |
 
 ```python
 from fmp import FmpClient

crates/fmp-py/python/fmp/_native/errors/__init__.pyi

@@ -64,6 +64,18 @@ class FmpError(builtins.Exception):
         Whether `body` was cut short to stay within the excerpt bound.
         """
 
+    @property
+    def decode_path(self) -> typing.Optional[builtins.str]:
+        r"""
+        Where a JSON response failed to decode, for example `"[37].beta"`; never the member value.
+        """
+
+    @property
+    def decode_kind(self) -> typing.Optional[builtins.str]:
+        r"""
+        Why a JSON response failed to decode: `"syntax"`, `"null"`, `"missing_member"`, `"wrong_type"`, or `"invalid_value"`.
+        """
+
 
 class FmpStatusError(FmpError):
     r"""

crates/fmp-py/src/errors.rs

@@ -64,6 +64,20 @@ static ERROR_ATTRIBUTES: &[MemberInfo] = &[
         default: None,
         deprecated: None,
     },
+    MemberInfo {
+        name: "decode_path",
+        r#type: <Option<String> as PyStubType>::type_output,
+        doc: "Where a JSON response failed to decode, for example `\"[37].beta\"`; never the member value.",
+        default: None,
+        deprecated: None,
+    },
+    MemberInfo {
+        name: "decode_kind",
+        r#type: <Option<String> as PyStubType>::type_output,
+        doc: "Why a JSON response failed to decode: `\"syntax\"`, `\"null\"`, `\"missing_member\"`, `\"wrong_type\"`, or `\"invalid_value\"`.",
+        default: None,
+        deprecated: None,
+    },
 ];
 
 /// Declares one exception type: the pyo3 type object under `fmp.errors` plus
@@ -188,6 +202,8 @@ fn set_validation_attributes(exception: &Bound<'_, PyBaseException>) -> PyResult
     exception.setattr("status", py.None())?;
     exception.setattr("body", py.None())?;
     exception.setattr("body_truncated", py.None())?;
+    exception.setattr("decode_path", py.None())?;
+    exception.setattr("decode_kind", py.None())?;
     Ok(())
 }
 
@@ -203,6 +219,13 @@ fn set_error_attributes(
         "body_truncated",
         error.body().map(libfmp::error::SafeBody::is_truncated),
     )?;
+    exception.setattr("decode_path", error.decode_path())?;
+    exception.setattr(
+        "decode_kind",
+        error
+            .decode_kind()
+            .map(libfmp::error::DecodeErrorKind::as_str),
+    )?;
     Ok(())
 }
 
@@ -219,6 +242,8 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
     base.setattr("status", py.None())?;
     base.setattr("body", py.None())?;
     base.setattr("body_truncated", py.None())?;
+    base.setattr("decode_path", py.None())?;
+    base.setattr("decode_kind", py.None())?;
 
     py.get_type::<FmpValidationError>()
         .setattr("category", "validation")?;

crates/fmp-py/tests/test_analyst.py

@@ -232,11 +232,13 @@ def test_stock_grades_summary_accepts_an_integral_float_count(client: Any, fixtu
 def test_stock_grades_summary_rejects_a_fractional_count(
     client: Any, fixture_server: FixtureServer, errors: SimpleNamespace
 ) -> None:
-    """A fractional count such as ``2.9`` maps to ``FmpDecodeError`` with the endpoint id."""
+    """A fractional count such as ``2.9`` maps to ``FmpDecodeError`` naming the endpoint and member."""
     fixture_server.route("/grades-consensus", [{**load_fixture("stock_grades_summary.json")[0], "buy": 2.9}])
     with pytest.raises(errors.FmpDecodeError) as raised:
         client.analyst.stock_grades_summary("AAPL")
     assert raised.value.endpoint == "grades-consensus"
+    assert raised.value.decode_path == "[0].buy"
+    assert raised.value.decode_kind == "invalid_value"
 
 
 def test_empty_array_decodes_to_an_empty_list(client: Any, fixture_server: FixtureServer) -> None:

crates/fmp-py/tests/test_import.py

@@ -245,3 +245,4 @@ def test_exception_attributes_default_to_none(errors: SimpleNamespace, name: str
     assert exception.args == ("message",)
     assert exception.category == category
     assert (exception.endpoint, exception.status, exception.body, exception.body_truncated) == (None, None, None, None)
+    assert (exception.decode_path, exception.decode_kind) == (None, None)

crates/fmp-py/tests/test_screener.py

@@ -213,9 +213,46 @@ def test_status_error_carries_the_screener_endpoint_id(
 def test_decode_error_for_a_non_json_body(client: Any, fixture_server: FixtureServer, errors: SimpleNamespace) -> None:
     """A body that is not JSON maps to ``FmpDecodeError`` rather than a raw exception."""
     fixture_server.route("/company-screener", b"not-json")
-    with pytest.raises(errors.FmpDecodeError):
+    with pytest.raises(errors.FmpDecodeError) as caught:
         client.screener.companies()
 
+    assert (caught.value.decode_kind, caught.value.decode_path) == ("syntax", None)
+
+
+def test_decode_error_names_the_null_member_row_and_member(
+    client: Any, fixture_server: FixtureServer, errors: SimpleNamespace
+) -> None:
+    """Issue #366: a ``null`` required member names its row index and member."""
+    fixture_server.route("/company-screener", load_fixture("company_screener_null_beta_synthetic.json"))
+    with pytest.raises(errors.FmpDecodeError) as caught:
+        client.screener.companies()
+
+    assert caught.value.decode_path == "[37].beta"
+    assert caught.value.decode_kind == "null"
+    assert str(caught.value).startswith(
+        "successful response could not be decoded: null value at [37].beta (endpoint: company-screener): "
+    )
+
+
+def test_decode_error_never_carries_a_string_member_value(
+    client: Any, fixture_server: FixtureServer, errors: SimpleNamespace
+) -> None:
+    """Issue #366: the offending string value reaches neither the message nor an attribute."""
+    sentinel = "SENTINEL-beta-text"
+    rows = load_fixture("company_screener_null_beta_synthetic.json")
+    rows[37]["beta"] = sentinel
+    fixture_server.route("/company-screener", rows)
+    with pytest.raises(errors.FmpDecodeError) as caught:
+        client.screener.companies()
+
+    error = caught.value
+    assert error.decode_path == "[37].beta"
+    assert error.decode_kind == "wrong_type"
+    assert error.body_truncated is True
+    assert sentinel not in str(error)
+    assert sentinel not in repr(error)
+    assert all(sentinel not in str(value) for value in vars(error).values())
+
 
 def test_companies_decodes_integral_float_and_fractional_market_caps(client: Any, fixture_server: FixtureServer) -> None:
     """Issue #339: an integral-float or fractional ``marketCap`` decodes as a ``float``."""

crates/libfmp/Cargo.toml

@@ -35,6 +35,7 @@ http.workspace = true
 reqwest.workspace = true
 serde.workspace = true
 serde_json.workspace = true
+serde_path_to_error.workspace = true
 tokio.workspace = true
 url.workspace = true
 

crates/libfmp/src/client.rs

@@ -489,13 +489,17 @@ impl Client {
                     response.body_bytes(),
                     ResponseMetadata::new(content_type, content_disposition),
                 )
-                .map_err(|_| {
+                .map_err(|failure| {
                     Error::decode(
                         Some(endpoint.id()),
                         Some(response.status()),
                         Some(safe_body(response.body(), redactor)),
                         "successful response could not be decoded",
                     )
+                    .with_decode_location(
+                        failure.path.map(|path| redactor.redact(&path)),
+                        failure.kind,
+                    )
                 });
         }
 

crates/libfmp/src/endpoints/mod.rs

@@ -38,7 +38,7 @@ use std::{fmt, marker::PhantomData};
 use bytes::Bytes;
 use serde::de::DeserializeOwned;
 
-use crate::transport::HttpMethod;
+use crate::{error::DecodeErrorKind, transport::HttpMethod};
 use metadata::EndpointMetadata;
 
 /// The JSON media type requested by FMP's structured-data endpoints.
@@ -335,7 +335,17 @@ impl<'a> ResponseMetadata<'a> {
     }
 }
 
-type Decoder<R> = for<'a> fn(Bytes, ResponseMetadata<'a>) -> std::result::Result<R, ()>;
+/// Where and why a response body failed to decode.
+///
+/// Only the member path and a coarse kind survive: the decoder's own message
+/// can quote the offending value, so it is classified and then dropped.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(crate) struct DecodeFailure {
+    pub(crate) path: Option<String>,
+    pub(crate) kind: DecodeErrorKind,
+}
+
+type Decoder<R> = for<'a> fn(Bytes, ResponseMetadata<'a>) -> std::result::Result<R, DecodeFailure>;
 
 /// Decoding and media-type expectations associated with one endpoint.
 pub struct ResponseContract<R> {
@@ -354,7 +364,7 @@ impl<R> ResponseContract<R> {
         &self,
         body: Bytes,
         metadata: ResponseMetadata<'_>,
-    ) -> std::result::Result<R, ()> {
+    ) -> std::result::Result<R, DecodeFailure> {
         (self.decoder)(body, metadata)
     }
 }
@@ -394,11 +404,56 @@ impl<R> fmt::Debug for ResponseContract<R> {
     }
 }
 
-fn decode_json<R>(body: Bytes, _metadata: ResponseMetadata<'_>) -> std::result::Result<R, ()>
+fn decode_json<R>(
+    body: Bytes,
+    _metadata: ResponseMetadata<'_>,
+) -> std::result::Result<R, DecodeFailure>
 where
     R: DeserializeOwned,
 {
-    serde_json::from_slice(&body).map_err(|_| ())
+    let mut deserializer = serde_json::Deserializer::from_slice(&body);
+    let value = serde_path_to_error::deserialize(&mut deserializer)
+        .map_err(|error| decode_failure(error.path(), error.inner()))?;
+    deserializer.end().map_err(|_| DecodeFailure {
+        path: None,
+        kind: DecodeErrorKind::Syntax,
+    })?;
+    Ok(value)
+}
+
+fn decode_failure(path: &serde_path_to_error::Path, error: &serde_json::Error) -> DecodeFailure {
+    let message = error.to_string();
+    let kind = match error.classify() {
+        serde_json::error::Category::Data => data_error_kind(&message),
+        _ => DecodeErrorKind::Syntax,
+    };
+    let mut path = (path.iter().next().is_some()).then(|| path.to_string());
+    if kind == DecodeErrorKind::MissingMember
+        && let Some(member) = missing_member_name(&message)
+    {
+        path = Some(match path {
+            Some(parent) => format!("{parent}.{member}"),
+            None => member.to_owned(),
+        });
+    }
+    DecodeFailure { path, kind }
+}
+
+fn data_error_kind(message: &str) -> DecodeErrorKind {
+    if message.starts_with("invalid type: null") {
+        DecodeErrorKind::Null
+    } else if message.starts_with("invalid type: ") {
+        DecodeErrorKind::WrongType
+    } else if message.starts_with("missing field ") {
+        DecodeErrorKind::MissingMember
+    } else {
+        DecodeErrorKind::InvalidValue
+    }
+}
+
+fn missing_member_name(message: &str) -> Option<&str> {
+    let name = message.strip_prefix("missing field `")?;
+    name.split_once('`').map(|(name, _)| name)
 }
 
 /// An owned binary response with validated media metadata.
@@ -466,7 +521,7 @@ pub type BinaryBody = BinaryResponse;
 fn decode_binary(
     body: Bytes,
     metadata: ResponseMetadata<'_>,
-) -> std::result::Result<BinaryResponse, ()> {
+) -> std::result::Result<BinaryResponse, DecodeFailure> {
     Ok(BinaryResponse {
         bytes: body,
         content_type: metadata.content_type.into(),