diff --git a/crates/fmp-py/examples/binary_and_dicts.py b/crates/fmp-py/examples/binary_and_dicts.py index ffa94b57..a8dc5c62 100644 --- a/crates/fmp-py/examples/binary_and_dicts.py +++ b/crates/fmp-py/examples/binary_and_dicts.py @@ -20,9 +20,9 @@ def save_xlsx(client: FmpClient) -> None: try: payload = client.statements.reports.xlsx("AAPL", 2024, "FY") except FmpDecodeError as error: - # A body whose Content-Type is not an XLSX (or generic binary) type is - # rejected as a decode error. For a binary endpoint error.body is raw bytes - # rendered as text, so log the structured attributes, not the body. + # A body that is neither labeled as XLSX (or generic binary) nor starts + # with the XLSX (ZIP) signature is rejected as a decode error. For a binary + # endpoint error.body is a short summary, never the raw bytes. print(f"xlsx rejected: category={error.category} status={error.status} endpoint={error.endpoint}") return print(f"{payload!r}: content_type={payload.content_type} bytes={payload.byte_len}") diff --git a/crates/libfmp/src/client.rs b/crates/libfmp/src/client.rs index 1dbfbe45..0dae892f 100644 --- a/crates/libfmp/src/client.rs +++ b/crates/libfmp/src/client.rs @@ -22,7 +22,9 @@ use url::Url; use crate::{ Result, VERSION, config::{Authentication, DEFAULT_BASE_URL, DEFAULT_PATH_PREFIX, RedirectPolicy}, - endpoints::{DecodeFailure, QueryEncoder, ResponseMetadata}, + endpoints::{ + DecodeFailure, ExpectedContentType, QueryEncoder, ResponseMetadata, is_provider_message, + }, error::{ConfigurationErrorKind, Error, Redactor, SafeBody, SecretString}, transport::{HttpExecutor, PreparedRequest, ReqwestExecutor, TransportResponse}, }; @@ -476,11 +478,12 @@ impl Client { return status_error(endpoint.id(), response, redactor); } + let expected = endpoint.response().expected_content_type(); let Some(content_type) = response.headers().get(CONTENT_TYPE) else { return Err(Error::decode( Some(endpoint.id()), Some(response.status()), - Some(safe_body(response.body(), redactor)), + Some(error_body(expected, response.body(), None, redactor)), "successful response omitted its content type", )); }; @@ -488,19 +491,30 @@ impl Client { return Err(Error::decode( Some(endpoint.id()), Some(response.status()), - Some(safe_body(response.body(), redactor)), + Some(error_body(expected, response.body(), None, redactor)), "successful response used an invalid content type", )); }; - if !endpoint - .response() - .expected_content_type() - .matches(content_type) - { + if !expected.matches(content_type) && !expected.matches_signature(response.body()) { + if expected.is_binary() + && ExpectedContentType::Json.matches(content_type) + && is_provider_message(response.body()) + { + return Err(Error::provider_message( + endpoint.id(), + response.status(), + Some(safe_body(response.body(), redactor)), + )); + } return Err(Error::decode( Some(endpoint.id()), Some(response.status()), - Some(safe_body(response.body(), redactor)), + Some(error_body( + expected, + response.body(), + Some(content_type), + redactor, + )), "successful response used an unexpected content type", )); } @@ -1055,6 +1069,27 @@ fn safe_body(body: &[u8], redactor: &Redactor) -> SafeBody { SafeBody::new(&String::from_utf8_lossy(body), redactor) } +/// Builds the body of a decode error, summarizing it for binary endpoints so +/// raw payload bytes never reach a diagnostic (#411). +fn error_body( + expected: ExpectedContentType, + body: &[u8], + content_type: Option<&str>, + redactor: &Redactor, +) -> SafeBody { + if !expected.is_binary() { + return safe_body(body, redactor); + } + let summary = match content_type { + Some(content_type) => format!( + "", + body.len() + ), + None => format!("", body.len()), + }; + SafeBody::new(&summary, redactor) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/libfmp/src/endpoints/mod.rs b/crates/libfmp/src/endpoints/mod.rs index abc2f163..e4bf795e 100644 --- a/crates/libfmp/src/endpoints/mod.rs +++ b/crates/libfmp/src/endpoints/mod.rs @@ -330,8 +330,38 @@ impl ExpectedContentType { .any(|expected| media_type.eq_ignore_ascii_case(expected)), } } + + pub(crate) const fn is_binary(self) -> bool { + matches!(self, Self::Binary(_)) + } + + /// Reports whether `body` opens with the signature of one of the expected + /// binary formats, whatever its declared media type. + /// + /// FMP has served real workbooks labeled `application/json` (#411). Only + /// the `Binary` arm sniffs; formats without a table entry never match. + pub(crate) fn matches_signature(self, body: &[u8]) -> bool { + let Self::Binary(expected) = self else { + return false; + }; + expected.iter().any(|expected| { + BINARY_SIGNATURES.iter().any(|(media_type, magic)| { + expected.eq_ignore_ascii_case(media_type) && body.starts_with(magic) + }) + }) + } } +const ZIP_MAGIC: &[u8] = b"PK\x03\x04"; + +const BINARY_SIGNATURES: &[(&str, &[u8])] = &[ + ( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ZIP_MAGIC, + ), + ("application/zip", ZIP_MAGIC), +]; + fn valid_json_media_type(media_type: &str) -> bool { let Some((type_name, subtype)) = media_type.split_once('/') else { return false; @@ -510,7 +540,7 @@ const ERROR_MESSAGE_MEMBER: &str = "Error Message"; /// One is a short plain-text line such as `Invalid name` that is not JSON at /// all; the other is a JSON object whose only member is `"Error Message"`. /// Both checks are bounded by size so ordinary payloads are not parsed twice. -fn is_provider_message(body: &[u8]) -> bool { +pub(crate) fn is_provider_message(body: &[u8]) -> bool { let body = body.trim_ascii(); match body.first() { None => false, diff --git a/crates/libfmp/src/endpoints/statements/reports.rs b/crates/libfmp/src/endpoints/statements/reports.rs index 235da1c8..b437b619 100644 --- a/crates/libfmp/src/endpoints/statements/reports.rs +++ b/crates/libfmp/src/endpoints/statements/reports.rs @@ -10,8 +10,11 @@ use crate::{ /// Official XLSX MIME followed by the binary fallback accepted for proxy compatibility. /// -/// This is an SDK interoperability policy. The endpoint does not inspect ZIP -/// magic and rejects every other or missing response content type. +/// This is an SDK interoperability policy. A body under any other content +/// type is still accepted when it starts with the ZIP magic `PK\x03\x04`, +/// because FMP labels real workbooks `application/json` (#411); its content +/// type is kept as received. A missing content type, or any other body, is +/// rejected without echoing the body bytes. pub const FINANCIAL_REPORTS_XLSX_CONTENT_TYPES: &[&str] = &[ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/octet-stream", diff --git a/crates/libfmp/tests/endpoints.rs b/crates/libfmp/tests/endpoints.rs index d8291da9..30002584 100644 --- a/crates/libfmp/tests/endpoints.rs +++ b/crates/libfmp/tests/endpoints.rs @@ -268,7 +268,7 @@ async fn binary_response_equality_includes_bytes_and_exact_content_metadata() { #[tokio::test] async fn binary_contract_rejects_missing_and_wrong_content_types() { const BINARY_TYPE: &str = "application/octet-stream"; - const BINARY_BYTES: &[u8] = b"future-binary-response"; + const BINARY_BYTES: &[u8] = b"future-binary-response\x00"; let endpoint: EndpointSpec<(), BinaryBody> = EndpointSpec::get_binary( "binary-contract-test", "future-download", @@ -322,7 +322,8 @@ async fn unexpected_binary_content_has_bounded_token_safe_diagnostics() { let diagnostic = format!("{error:?} {error}"); assert_eq!(error.category(), ErrorCategory::Decode); - assert!(safe_body.is_truncated()); + assert!(!safe_body.is_truncated()); + assert!(safe_body.as_str().starts_with(" libfmp::Result { + let client = Client::builder() + .authentication(Authentication::fmp_header("secret")) + .executor(Arc::new(FixtureExecutor::new([response]))) + .build() + .unwrap(); + client + .financial_reports_xlsx(FinancialReportsXlsxQuery::new( + Ticker::new("AAPL").unwrap(), + Year(2024), + FiscalPeriod::FullYear, + )) + .await +} + +#[tokio::test] +async fn xlsx_accepts_a_zip_workbook_labeled_as_json_and_keeps_its_content_type() { + const DISPOSITION: &str = "attachment; filename = AAPL_2024_FY_.xlsx"; + let official = xlsx_call(binary_fixture(Some(OFFICIAL_XLSX), None, ZIP_WORKBOOK)) + .await + .unwrap(); + let labeled = xlsx_call(binary_fixture( + Some(LABELED_JSON), + Some(DISPOSITION), + ZIP_WORKBOOK, + )) + .await + .unwrap(); + + assert_eq!(official.as_bytes(), ZIP_WORKBOOK); + assert_eq!(official.content_type(), OFFICIAL_XLSX); + assert_eq!(labeled.as_bytes(), ZIP_WORKBOOK); + assert_eq!(labeled.content_type(), LABELED_JSON); + assert_eq!(labeled.content_disposition(), Some(DISPOSITION)); +} + +#[tokio::test] +async fn xlsx_labeled_as_json_keeps_the_provider_message_check() { + const MESSAGE: &[u8] = br#"{"Error Message":"Limit reached for xlsx"}"#; + let error = xlsx_call(binary_fixture(Some(LABELED_JSON), None, MESSAGE)) + .await + .unwrap_err(); + + assert_eq!(error.category(), ErrorCategory::Status); + assert_eq!(error.status_code(), Some(200)); + assert_eq!(error.endpoint(), Some("financial-reports-xlsx")); + assert_eq!( + error.body().unwrap().as_str(), + r#"{"Error Message":"Limit reached for xlsx"}"# + ); +} + +#[tokio::test] +async fn xlsx_rejects_other_bodies_without_echoing_their_bytes() { + const GARBAGE: &[u8] = b"\x00\x01garbage-private-payload\xff"; + let error = xlsx_call(binary_fixture(Some(LABELED_JSON), None, GARBAGE)) + .await + .unwrap_err(); + let diagnostic = format!("{error:?} {error}"); + + assert_eq!(error.category(), ErrorCategory::Decode); + assert_eq!(error.status_code(), Some(200)); + assert_eq!( + error.body().unwrap().as_str(), + format!( + "", + GARBAGE.len() + ) + ); + assert!(!diagnostic.contains("garbage-private-payload")); +} + +#[tokio::test] +async fn json_endpoints_never_sniff_workbook_signatures() { + let executor = Arc::new(FixtureExecutor::new([ + binary_fixture(Some(OFFICIAL_XLSX), None, ZIP_WORKBOOK), + binary_fixture(Some("application/json"), None, ZIP_WORKBOOK), + ])); + let client = Client::builder() + .authentication(Authentication::fmp_header("secret")) + .executor(executor) + .build() + .unwrap(); + let query = || { + FinancialReportsJsonQuery::new( + Ticker::new("AAPL").unwrap(), + Year(2024), + FiscalPeriod::FullYear, + ) + }; + + for _ in 0..2 { + let error = client.financial_reports_json(query()).await.unwrap_err(); + assert_eq!(error.category(), ErrorCategory::Decode); + assert_eq!(error.endpoint(), Some("financial-reports-json")); + } +} diff --git a/docs/adr/0002-shared-wire-types-and-python-conversions.md b/docs/adr/0002-shared-wire-types-and-python-conversions.md index b8799c8d..f0c8c73e 100644 --- a/docs/adr/0002-shared-wire-types-and-python-conversions.md +++ b/docs/adr/0002-shared-wire-types-and-python-conversions.md @@ -87,7 +87,9 @@ contents, or content-type parameters. Each endpoint owns its accepted MIME allow-list because the provider source does not establish one global binary MIME policy. The client validates `Content-Type` against that endpoint contract before constructing `BinaryResponse`; optional -`Content-Disposition` is retained only when it is valid header text. +`Content-Disposition` is retained only when it is valid header text. Since #411 +a body with the magic bytes of an expected format is also accepted under a +mismatched content type (ADR 0008). `PartialEq` and `Eq` compare bytes, exact content type, and optional content disposition. Metadata is part of binary response identity rather than incidental diff --git a/docs/adr/0008-as-reported-and-financial-reports-api-names.md b/docs/adr/0008-as-reported-and-financial-reports-api-names.md index ae77c103..115456cb 100644 --- a/docs/adr/0008-as-reported-and-financial-reports-api-names.md +++ b/docs/adr/0008-as-reported-and-financial-reports-api-names.md @@ -42,6 +42,17 @@ though the endpoint is documented as an XLSX download. The SDK therefore uses `application/octet-stream` for binary-compatible proxies; it does not model the copied JSON example as the XLSX response contract. +Update 2026-10-04 (#411): FMP now serves the real workbook with +`Content-Type: application/json; charset=utf-8`. The SDK (Rust and Go) also +accepts a successful body that starts with the magic bytes of an expected +format (XLSX and the ZIP family: `PK\x03\x04`), whatever its declared media +type; the content type is kept exactly as received. `Content-Disposition` is +not used for the decision. A JSON-labeled body that is an FMP provider +message still yields the provider-message error. Every other error on a +binary endpoint summarises the body (``) +instead of echoing its bytes. `application/octet-stream` has no signature and +is never sniffed. + `FinancialReportDate` links use `SecretUrl` so embedded API keys remain redacted in debug and display surfaces; implicit serialization is disallowed. Revenue segmentation uses the shared dynamic row type and the closed documented query value diff --git a/sdk/go/binary.go b/sdk/go/binary.go index 296348b3..157a4a9d 100644 --- a/sdk/go/binary.go +++ b/sdk/go/binary.go @@ -1,6 +1,7 @@ package fmp import ( + "bytes" "context" "fmt" "strings" @@ -32,7 +33,10 @@ func (p BinaryPayload) Format(f fmt.State, _ rune) { // getBinary issues one GET for a logical endpoint and returns the raw body // when the response carries one of the expected media types, compared -// case-insensitively without parameters. endpointID is the registry id +// case-insensitively without parameters, or when the body opens with the +// signature of one of them (FMP has labeled real workbooks as JSON, #411). +// A JSON-labeled provider message keeps its provider-message error; every +// other error summarizes the body instead of echoing its bytes. endpointID is the registry id // carried by errors, never a URL. The redirect, timeout, body cap, and // no-retry rules of getJSON apply unchanged. func (c *Client) getBinary(ctx context.Context, endpointID, relativePath string, query []queryParam, @@ -57,13 +61,16 @@ func (c *Client) getBinary(ctx context.Context, endpointID, relativePath string, contentType := resp.header.Get("Content-Type") switch { case contentType == "": - return BinaryPayload{}, decodeError(endpointID, resp.status, c.safeBody(resp.body), + return BinaryPayload{}, decodeError(endpointID, resp.status, c.binaryErrorBody(resp.body, ""), "successful response omitted its content type", nil) case !visibleHeaderText(contentType): - return BinaryPayload{}, decodeError(endpointID, resp.status, c.safeBody(resp.body), + return BinaryPayload{}, decodeError(endpointID, resp.status, c.binaryErrorBody(resp.body, ""), "successful response used an invalid content type", nil) - case !matchesMediaType(contentType, expectedContentTypes): - return BinaryPayload{}, decodeError(endpointID, resp.status, c.safeBody(resp.body), + case !matchesMediaType(contentType, expectedContentTypes) && !matchesSignature(resp.body, expectedContentTypes): + if validJSONMediaType(contentType) && isProviderMessage(resp.body) { + return BinaryPayload{}, providerMessageError(endpointID, resp.status, c.safeBody(resp.body)) + } + return BinaryPayload{}, decodeError(endpointID, resp.status, c.binaryErrorBody(resp.body, contentType), "successful response used an unexpected content type", nil) } disposition := resp.header.Get("Content-Disposition") @@ -87,3 +94,31 @@ func matchesMediaType(contentType string, expected []string) bool { } return false } + +var zipMagic = []byte("PK\x03\x04") + +// binarySignatures maps binary media types to the magic bytes their bodies +// start with. Types without an entry are never sniffed. +var binarySignatures = map[string][]byte{ + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": zipMagic, + "application/zip": zipMagic, +} + +func matchesSignature(body []byte, expected []string) bool { + for _, candidate := range expected { + if magic, ok := binarySignatures[strings.ToLower(candidate)]; ok && bytes.HasPrefix(body, magic) { + return true + } + } + return false +} + +// binaryErrorBody summarizes a rejected binary body without its bytes. +func (c *Client) binaryErrorBody(body []byte, contentType string) *SafeBody { + summary := fmt.Sprintf("", len(body)) + if contentType != "" { + summary = fmt.Sprintf("", len(body), contentType) + } + safe := NewSafeBody(summary, c.redactor) + return &safe +} diff --git a/sdk/go/binary_test.go b/sdk/go/binary_test.go index 203487ae..2aba41b0 100644 --- a/sdk/go/binary_test.go +++ b/sdk/go/binary_test.go @@ -95,7 +95,7 @@ func TestGetBinaryRetainsBytesAndSafeContentMetadata(t *testing.T) { // binary_contract_rejects_missing_and_wrong_content_types. func TestGetBinaryRejectsMissingAndWrongContentTypes(t *testing.T) { t.Parallel() - const payload = "future-binary-response" + const payload = "future-binary-response\x00" cases := []struct { name string handler http.HandlerFunc @@ -117,7 +117,8 @@ func TestGetBinaryRejectsMissingAndWrongContentTypes(t *testing.T) { server, rec := newServer(t, tc.handler) _, err := getBinaryProbe(t, newClient(t, server)) typed := assertBinaryError(t, err, CategoryDecode, http.StatusOK) - if typed.Message != tc.message || typed.Body == nil || typed.Body.Text != payload || rec.count() != 1 { + if typed.Message != tc.message || typed.Body == nil || strings.Contains(typed.Body.Text, "future-binary") || + !strings.HasPrefix(typed.Body.Text, " MaxSafeBodyBytes { + if typed.Body == nil || typed.Body.Truncated || !strings.HasPrefix(typed.Body.Text, "", len(garbage), labeledJSON) + if typed.Body == nil || typed.Body.Text != want { + t.Fatalf("body = %+v, want %q", typed.Body, want) + } + if diagnostic := fmt.Sprintf("%v %+v %s", typed, typed, err.Error()); strings.Contains(diagnostic, "garbage-private") { + t.Fatalf("diagnostic leaked the body: %s", diagnostic) + } +} + +// TestJSONEndpointsNeverSniffWorkbookSignatures mirrors +// json_endpoints_never_sniff_workbook_signatures. +func TestJSONEndpointsNeverSniffWorkbookSignatures(t *testing.T) { + t.Parallel() + for _, contentType := range []string{statementsXlsxType, "application/json"} { + server, _ := newServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", contentType) + _, _ = w.Write([]byte(zipWorkbook)) + }) + _, err := newClient(t, server).Statements.Reports.JSON(context.Background(), + NewFinancialReportsJSONQuery("AAPL", 2024, FiscalPeriodFullYear)) + var typed *Error + if !errors.As(err, &typed) || typed.Category != CategoryDecode || typed.Endpoint != "financial-reports-json" { + t.Fatalf("%s: error = %+v (%v)", contentType, typed, err) + } + } +}