From a595159355b11a4a1c21faffd87436c1ba8323f4 Mon Sep 17 00:00:00 2001 From: benbenbang Date: Sun, 4 Oct 2026 11:00:12 +0800 Subject: [PATCH 1/3] fix(sdk/go): accept xlsx bodies FMP labels as JSON FMP has labeled real workbooks as JSON (#411), causing getBinary to reject valid responses whose content type did not match the expected media types. Accept a body when it opens with the signature of one of the expected types, sniffing the ZIP magic bytes for xlsx and zip media types. JSON endpoints are never sniffed, so workbook signatures on JSON responses still fail as before. A JSON-labeled provider message keeps its provider-message error; every other rejected binary body is now summarized as a byte count and content type rather than echoed, preventing private payload bytes from leaking into error diagnostics. --- sdk/go/binary.go | 45 ++++++++++++++++++--- sdk/go/binary_test.go | 91 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 128 insertions(+), 8 deletions(-) diff --git a/sdk/go/binary.go b/sdk/go/binary.go index 296348b3..157a4a9d 100644 --- a/sdk/go/binary.go +++ b/sdk/go/binary.go @@ -1,6 +1,7 @@ package fmp import ( + "bytes" "context" "fmt" "strings" @@ -32,7 +33,10 @@ func (p BinaryPayload) Format(f fmt.State, _ rune) { // getBinary issues one GET for a logical endpoint and returns the raw body // when the response carries one of the expected media types, compared -// case-insensitively without parameters. endpointID is the registry id +// case-insensitively without parameters, or when the body opens with the +// signature of one of them (FMP has labeled real workbooks as JSON, #411). +// A JSON-labeled provider message keeps its provider-message error; every +// other error summarizes the body instead of echoing its bytes. endpointID is the registry id // carried by errors, never a URL. The redirect, timeout, body cap, and // no-retry rules of getJSON apply unchanged. func (c *Client) getBinary(ctx context.Context, endpointID, relativePath string, query []queryParam, @@ -57,13 +61,16 @@ func (c *Client) getBinary(ctx context.Context, endpointID, relativePath string, contentType := resp.header.Get("Content-Type") switch { case contentType == "": - return BinaryPayload{}, decodeError(endpointID, resp.status, c.safeBody(resp.body), + return BinaryPayload{}, decodeError(endpointID, resp.status, c.binaryErrorBody(resp.body, ""), "successful response omitted its content type", nil) case !visibleHeaderText(contentType): - return BinaryPayload{}, decodeError(endpointID, resp.status, c.safeBody(resp.body), + return BinaryPayload{}, decodeError(endpointID, resp.status, c.binaryErrorBody(resp.body, ""), "successful response used an invalid content type", nil) - case !matchesMediaType(contentType, expectedContentTypes): - return BinaryPayload{}, decodeError(endpointID, resp.status, c.safeBody(resp.body), + case !matchesMediaType(contentType, expectedContentTypes) && !matchesSignature(resp.body, expectedContentTypes): + if validJSONMediaType(contentType) && isProviderMessage(resp.body) { + return BinaryPayload{}, providerMessageError(endpointID, resp.status, c.safeBody(resp.body)) + } + return BinaryPayload{}, decodeError(endpointID, resp.status, c.binaryErrorBody(resp.body, contentType), "successful response used an unexpected content type", nil) } disposition := resp.header.Get("Content-Disposition") @@ -87,3 +94,31 @@ func matchesMediaType(contentType string, expected []string) bool { } return false } + +var zipMagic = []byte("PK\x03\x04") + +// binarySignatures maps binary media types to the magic bytes their bodies +// start with. Types without an entry are never sniffed. +var binarySignatures = map[string][]byte{ + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": zipMagic, + "application/zip": zipMagic, +} + +func matchesSignature(body []byte, expected []string) bool { + for _, candidate := range expected { + if magic, ok := binarySignatures[strings.ToLower(candidate)]; ok && bytes.HasPrefix(body, magic) { + return true + } + } + return false +} + +// binaryErrorBody summarizes a rejected binary body without its bytes. +func (c *Client) binaryErrorBody(body []byte, contentType string) *SafeBody { + summary := fmt.Sprintf("", len(body)) + if contentType != "" { + summary = fmt.Sprintf("", len(body), contentType) + } + safe := NewSafeBody(summary, c.redactor) + return &safe +} diff --git a/sdk/go/binary_test.go b/sdk/go/binary_test.go index 203487ae..2aba41b0 100644 --- a/sdk/go/binary_test.go +++ b/sdk/go/binary_test.go @@ -95,7 +95,7 @@ func TestGetBinaryRetainsBytesAndSafeContentMetadata(t *testing.T) { // binary_contract_rejects_missing_and_wrong_content_types. func TestGetBinaryRejectsMissingAndWrongContentTypes(t *testing.T) { t.Parallel() - const payload = "future-binary-response" + const payload = "future-binary-response\x00" cases := []struct { name string handler http.HandlerFunc @@ -117,7 +117,8 @@ func TestGetBinaryRejectsMissingAndWrongContentTypes(t *testing.T) { server, rec := newServer(t, tc.handler) _, err := getBinaryProbe(t, newClient(t, server)) typed := assertBinaryError(t, err, CategoryDecode, http.StatusOK) - if typed.Message != tc.message || typed.Body == nil || typed.Body.Text != payload || rec.count() != 1 { + if typed.Message != tc.message || typed.Body == nil || strings.Contains(typed.Body.Text, "future-binary") || + !strings.HasPrefix(typed.Body.Text, " MaxSafeBodyBytes { + if typed.Body == nil || typed.Body.Truncated || !strings.HasPrefix(typed.Body.Text, "", len(garbage), labeledJSON) + if typed.Body == nil || typed.Body.Text != want { + t.Fatalf("body = %+v, want %q", typed.Body, want) + } + if diagnostic := fmt.Sprintf("%v %+v %s", typed, typed, err.Error()); strings.Contains(diagnostic, "garbage-private") { + t.Fatalf("diagnostic leaked the body: %s", diagnostic) + } +} + +// TestJSONEndpointsNeverSniffWorkbookSignatures mirrors +// json_endpoints_never_sniff_workbook_signatures. +func TestJSONEndpointsNeverSniffWorkbookSignatures(t *testing.T) { + t.Parallel() + for _, contentType := range []string{statementsXlsxType, "application/json"} { + server, _ := newServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", contentType) + _, _ = w.Write([]byte(zipWorkbook)) + }) + _, err := newClient(t, server).Statements.Reports.JSON(context.Background(), + NewFinancialReportsJSONQuery("AAPL", 2024, FiscalPeriodFullYear)) + var typed *Error + if !errors.As(err, &typed) || typed.Category != CategoryDecode || typed.Endpoint != "financial-reports-json" { + t.Fatalf("%s: error = %+v (%v)", contentType, typed, err) + } + } +} From 88b2615a70974b0a51232e5dbe19f8b13ddf49eb Mon Sep 17 00:00:00 2001 From: benbenbang Date: Sun, 4 Oct 2026 11:00:42 +0800 Subject: [PATCH 2/3] fix(client): accept xlsx bodies labeled as json via zip magic sniff FMP serves real XLSX workbooks under `application/json` (#411), which the strict content-type contract rejected. Binary endpoints now sniff the ZIP magic `PK\x03\x04` and accept a matching body under any content type, keeping the content type as received. The provider-message check still runs first, so a JSON error object such as `{"Error Message":...}` served on a binary endpoint is surfaced as a provider message rather than decoded as a workbook. Decode errors on binary endpoints now summarize the body as `` so raw payload bytes never reach a diagnostic. JSON endpoints never sniff workbook signatures. --- crates/libfmp/src/client.rs | 53 +++++++-- crates/libfmp/src/endpoints/mod.rs | 32 +++++- .../src/endpoints/statements/reports.rs | 7 +- crates/libfmp/tests/endpoints.rs | 5 +- .../tests/statements_report_endpoints.rs | 104 +++++++++++++++++- 5 files changed, 186 insertions(+), 15 deletions(-) diff --git a/crates/libfmp/src/client.rs b/crates/libfmp/src/client.rs index 1dbfbe45..0dae892f 100644 --- a/crates/libfmp/src/client.rs +++ b/crates/libfmp/src/client.rs @@ -22,7 +22,9 @@ use url::Url; use crate::{ Result, VERSION, config::{Authentication, DEFAULT_BASE_URL, DEFAULT_PATH_PREFIX, RedirectPolicy}, - endpoints::{DecodeFailure, QueryEncoder, ResponseMetadata}, + endpoints::{ + DecodeFailure, ExpectedContentType, QueryEncoder, ResponseMetadata, is_provider_message, + }, error::{ConfigurationErrorKind, Error, Redactor, SafeBody, SecretString}, transport::{HttpExecutor, PreparedRequest, ReqwestExecutor, TransportResponse}, }; @@ -476,11 +478,12 @@ impl Client { return status_error(endpoint.id(), response, redactor); } + let expected = endpoint.response().expected_content_type(); let Some(content_type) = response.headers().get(CONTENT_TYPE) else { return Err(Error::decode( Some(endpoint.id()), Some(response.status()), - Some(safe_body(response.body(), redactor)), + Some(error_body(expected, response.body(), None, redactor)), "successful response omitted its content type", )); }; @@ -488,19 +491,30 @@ impl Client { return Err(Error::decode( Some(endpoint.id()), Some(response.status()), - Some(safe_body(response.body(), redactor)), + Some(error_body(expected, response.body(), None, redactor)), "successful response used an invalid content type", )); }; - if !endpoint - .response() - .expected_content_type() - .matches(content_type) - { + if !expected.matches(content_type) && !expected.matches_signature(response.body()) { + if expected.is_binary() + && ExpectedContentType::Json.matches(content_type) + && is_provider_message(response.body()) + { + return Err(Error::provider_message( + endpoint.id(), + response.status(), + Some(safe_body(response.body(), redactor)), + )); + } return Err(Error::decode( Some(endpoint.id()), Some(response.status()), - Some(safe_body(response.body(), redactor)), + Some(error_body( + expected, + response.body(), + Some(content_type), + redactor, + )), "successful response used an unexpected content type", )); } @@ -1055,6 +1069,27 @@ fn safe_body(body: &[u8], redactor: &Redactor) -> SafeBody { SafeBody::new(&String::from_utf8_lossy(body), redactor) } +/// Builds the body of a decode error, summarizing it for binary endpoints so +/// raw payload bytes never reach a diagnostic (#411). +fn error_body( + expected: ExpectedContentType, + body: &[u8], + content_type: Option<&str>, + redactor: &Redactor, +) -> SafeBody { + if !expected.is_binary() { + return safe_body(body, redactor); + } + let summary = match content_type { + Some(content_type) => format!( + "", + body.len() + ), + None => format!("", body.len()), + }; + SafeBody::new(&summary, redactor) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/libfmp/src/endpoints/mod.rs b/crates/libfmp/src/endpoints/mod.rs index abc2f163..e4bf795e 100644 --- a/crates/libfmp/src/endpoints/mod.rs +++ b/crates/libfmp/src/endpoints/mod.rs @@ -330,8 +330,38 @@ impl ExpectedContentType { .any(|expected| media_type.eq_ignore_ascii_case(expected)), } } + + pub(crate) const fn is_binary(self) -> bool { + matches!(self, Self::Binary(_)) + } + + /// Reports whether `body` opens with the signature of one of the expected + /// binary formats, whatever its declared media type. + /// + /// FMP has served real workbooks labeled `application/json` (#411). Only + /// the `Binary` arm sniffs; formats without a table entry never match. + pub(crate) fn matches_signature(self, body: &[u8]) -> bool { + let Self::Binary(expected) = self else { + return false; + }; + expected.iter().any(|expected| { + BINARY_SIGNATURES.iter().any(|(media_type, magic)| { + expected.eq_ignore_ascii_case(media_type) && body.starts_with(magic) + }) + }) + } } +const ZIP_MAGIC: &[u8] = b"PK\x03\x04"; + +const BINARY_SIGNATURES: &[(&str, &[u8])] = &[ + ( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ZIP_MAGIC, + ), + ("application/zip", ZIP_MAGIC), +]; + fn valid_json_media_type(media_type: &str) -> bool { let Some((type_name, subtype)) = media_type.split_once('/') else { return false; @@ -510,7 +540,7 @@ const ERROR_MESSAGE_MEMBER: &str = "Error Message"; /// One is a short plain-text line such as `Invalid name` that is not JSON at /// all; the other is a JSON object whose only member is `"Error Message"`. /// Both checks are bounded by size so ordinary payloads are not parsed twice. -fn is_provider_message(body: &[u8]) -> bool { +pub(crate) fn is_provider_message(body: &[u8]) -> bool { let body = body.trim_ascii(); match body.first() { None => false, diff --git a/crates/libfmp/src/endpoints/statements/reports.rs b/crates/libfmp/src/endpoints/statements/reports.rs index 235da1c8..b437b619 100644 --- a/crates/libfmp/src/endpoints/statements/reports.rs +++ b/crates/libfmp/src/endpoints/statements/reports.rs @@ -10,8 +10,11 @@ use crate::{ /// Official XLSX MIME followed by the binary fallback accepted for proxy compatibility. /// -/// This is an SDK interoperability policy. The endpoint does not inspect ZIP -/// magic and rejects every other or missing response content type. +/// This is an SDK interoperability policy. A body under any other content +/// type is still accepted when it starts with the ZIP magic `PK\x03\x04`, +/// because FMP labels real workbooks `application/json` (#411); its content +/// type is kept as received. A missing content type, or any other body, is +/// rejected without echoing the body bytes. pub const FINANCIAL_REPORTS_XLSX_CONTENT_TYPES: &[&str] = &[ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/octet-stream", diff --git a/crates/libfmp/tests/endpoints.rs b/crates/libfmp/tests/endpoints.rs index d8291da9..30002584 100644 --- a/crates/libfmp/tests/endpoints.rs +++ b/crates/libfmp/tests/endpoints.rs @@ -268,7 +268,7 @@ async fn binary_response_equality_includes_bytes_and_exact_content_metadata() { #[tokio::test] async fn binary_contract_rejects_missing_and_wrong_content_types() { const BINARY_TYPE: &str = "application/octet-stream"; - const BINARY_BYTES: &[u8] = b"future-binary-response"; + const BINARY_BYTES: &[u8] = b"future-binary-response\x00"; let endpoint: EndpointSpec<(), BinaryBody> = EndpointSpec::get_binary( "binary-contract-test", "future-download", @@ -322,7 +322,8 @@ async fn unexpected_binary_content_has_bounded_token_safe_diagnostics() { let diagnostic = format!("{error:?} {error}"); assert_eq!(error.category(), ErrorCategory::Decode); - assert!(safe_body.is_truncated()); + assert!(!safe_body.is_truncated()); + assert!(safe_body.as_str().starts_with(" libfmp::Result { + let client = Client::builder() + .authentication(Authentication::fmp_header("secret")) + .executor(Arc::new(FixtureExecutor::new([response]))) + .build() + .unwrap(); + client + .financial_reports_xlsx(FinancialReportsXlsxQuery::new( + Ticker::new("AAPL").unwrap(), + Year(2024), + FiscalPeriod::FullYear, + )) + .await +} + +#[tokio::test] +async fn xlsx_accepts_a_zip_workbook_labeled_as_json_and_keeps_its_content_type() { + const DISPOSITION: &str = "attachment; filename = AAPL_2024_FY_.xlsx"; + let official = xlsx_call(binary_fixture(Some(OFFICIAL_XLSX), None, ZIP_WORKBOOK)) + .await + .unwrap(); + let labeled = xlsx_call(binary_fixture( + Some(LABELED_JSON), + Some(DISPOSITION), + ZIP_WORKBOOK, + )) + .await + .unwrap(); + + assert_eq!(official.as_bytes(), ZIP_WORKBOOK); + assert_eq!(official.content_type(), OFFICIAL_XLSX); + assert_eq!(labeled.as_bytes(), ZIP_WORKBOOK); + assert_eq!(labeled.content_type(), LABELED_JSON); + assert_eq!(labeled.content_disposition(), Some(DISPOSITION)); +} + +#[tokio::test] +async fn xlsx_labeled_as_json_keeps_the_provider_message_check() { + const MESSAGE: &[u8] = br#"{"Error Message":"Limit reached for xlsx"}"#; + let error = xlsx_call(binary_fixture(Some(LABELED_JSON), None, MESSAGE)) + .await + .unwrap_err(); + + assert_eq!(error.category(), ErrorCategory::Status); + assert_eq!(error.status_code(), Some(200)); + assert_eq!(error.endpoint(), Some("financial-reports-xlsx")); + assert_eq!( + error.body().unwrap().as_str(), + r#"{"Error Message":"Limit reached for xlsx"}"# + ); +} + +#[tokio::test] +async fn xlsx_rejects_other_bodies_without_echoing_their_bytes() { + const GARBAGE: &[u8] = b"\x00\x01garbage-private-payload\xff"; + let error = xlsx_call(binary_fixture(Some(LABELED_JSON), None, GARBAGE)) + .await + .unwrap_err(); + let diagnostic = format!("{error:?} {error}"); + + assert_eq!(error.category(), ErrorCategory::Decode); + assert_eq!(error.status_code(), Some(200)); + assert_eq!( + error.body().unwrap().as_str(), + format!( + "", + GARBAGE.len() + ) + ); + assert!(!diagnostic.contains("garbage-private-payload")); +} + +#[tokio::test] +async fn json_endpoints_never_sniff_workbook_signatures() { + let executor = Arc::new(FixtureExecutor::new([ + binary_fixture(Some(OFFICIAL_XLSX), None, ZIP_WORKBOOK), + binary_fixture(Some("application/json"), None, ZIP_WORKBOOK), + ])); + let client = Client::builder() + .authentication(Authentication::fmp_header("secret")) + .executor(executor) + .build() + .unwrap(); + let query = || { + FinancialReportsJsonQuery::new( + Ticker::new("AAPL").unwrap(), + Year(2024), + FiscalPeriod::FullYear, + ) + }; + + for _ in 0..2 { + let error = client.financial_reports_json(query()).await.unwrap_err(); + assert_eq!(error.category(), ErrorCategory::Decode); + assert_eq!(error.endpoint(), Some("financial-reports-json")); + } +} From 5ddd08bb01ef99d9ea5539376eab133f1e0278ba Mon Sep 17 00:00:00 2001 From: benbenbang Date: Sun, 4 Oct 2026 11:04:45 +0800 Subject: [PATCH 3/3] docs(adr): record xlsx content-type relaxation and binary error summaries Document the #411 change where FMP serves the real XLSX workbook under a JSON content type. The SDK now accepts a successful body that starts with the magic bytes of an expected format (XLSX and the ZIP family `PK\x03\x04`) regardless of its declared media type, keeping the content type exactly as received. `Content-Disposition` is not used for the decision and `application/octet-stream` is never sniffed. Record that every other error on a binary endpoint summarises the body (``) instead of echoing raw bytes, and update the example comment to match. ADR 0002 cross-references ADR 0008 for the relaxed content-type policy. close #411 --- crates/fmp-py/examples/binary_and_dicts.py | 6 +++--- .../0002-shared-wire-types-and-python-conversions.md | 4 +++- ...008-as-reported-and-financial-reports-api-names.md | 11 +++++++++++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/crates/fmp-py/examples/binary_and_dicts.py b/crates/fmp-py/examples/binary_and_dicts.py index ffa94b57..a8dc5c62 100644 --- a/crates/fmp-py/examples/binary_and_dicts.py +++ b/crates/fmp-py/examples/binary_and_dicts.py @@ -20,9 +20,9 @@ def save_xlsx(client: FmpClient) -> None: try: payload = client.statements.reports.xlsx("AAPL", 2024, "FY") except FmpDecodeError as error: - # A body whose Content-Type is not an XLSX (or generic binary) type is - # rejected as a decode error. For a binary endpoint error.body is raw bytes - # rendered as text, so log the structured attributes, not the body. + # A body that is neither labeled as XLSX (or generic binary) nor starts + # with the XLSX (ZIP) signature is rejected as a decode error. For a binary + # endpoint error.body is a short summary, never the raw bytes. print(f"xlsx rejected: category={error.category} status={error.status} endpoint={error.endpoint}") return print(f"{payload!r}: content_type={payload.content_type} bytes={payload.byte_len}") diff --git a/docs/adr/0002-shared-wire-types-and-python-conversions.md b/docs/adr/0002-shared-wire-types-and-python-conversions.md index b8799c8d..f0c8c73e 100644 --- a/docs/adr/0002-shared-wire-types-and-python-conversions.md +++ b/docs/adr/0002-shared-wire-types-and-python-conversions.md @@ -87,7 +87,9 @@ contents, or content-type parameters. Each endpoint owns its accepted MIME allow-list because the provider source does not establish one global binary MIME policy. The client validates `Content-Type` against that endpoint contract before constructing `BinaryResponse`; optional -`Content-Disposition` is retained only when it is valid header text. +`Content-Disposition` is retained only when it is valid header text. Since #411 +a body with the magic bytes of an expected format is also accepted under a +mismatched content type (ADR 0008). `PartialEq` and `Eq` compare bytes, exact content type, and optional content disposition. Metadata is part of binary response identity rather than incidental diff --git a/docs/adr/0008-as-reported-and-financial-reports-api-names.md b/docs/adr/0008-as-reported-and-financial-reports-api-names.md index ae77c103..115456cb 100644 --- a/docs/adr/0008-as-reported-and-financial-reports-api-names.md +++ b/docs/adr/0008-as-reported-and-financial-reports-api-names.md @@ -42,6 +42,17 @@ though the endpoint is documented as an XLSX download. The SDK therefore uses `application/octet-stream` for binary-compatible proxies; it does not model the copied JSON example as the XLSX response contract. +Update 2026-10-04 (#411): FMP now serves the real workbook with +`Content-Type: application/json; charset=utf-8`. The SDK (Rust and Go) also +accepts a successful body that starts with the magic bytes of an expected +format (XLSX and the ZIP family: `PK\x03\x04`), whatever its declared media +type; the content type is kept exactly as received. `Content-Disposition` is +not used for the decision. A JSON-labeled body that is an FMP provider +message still yields the provider-message error. Every other error on a +binary endpoint summarises the body (``) +instead of echoing its bytes. `application/octet-stream` has no signature and +is never sniffed. + `FinancialReportDate` links use `SecretUrl` so embedded API keys remain redacted in debug and display surfaces; implicit serialization is disallowed. Revenue segmentation uses the shared dynamic row type and the closed documented query value