Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions crates/fmp-py/examples/binary_and_dicts.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@ def save_xlsx(client: FmpClient) -> None:
try:
payload = client.statements.reports.xlsx("AAPL", 2024, "FY")
except FmpDecodeError as error:
# A body whose Content-Type is not an XLSX (or generic binary) type is
# rejected as a decode error. For a binary endpoint error.body is raw bytes
# rendered as text, so log the structured attributes, not the body.
# A body that is neither labeled as XLSX (or generic binary) nor starts
# with the XLSX (ZIP) signature is rejected as a decode error. For a binary
# endpoint error.body is a short summary, never the raw bytes.
print(f"xlsx rejected: category={error.category} status={error.status} endpoint={error.endpoint}")
return
print(f"{payload!r}: content_type={payload.content_type} bytes={payload.byte_len}")
Expand Down
53 changes: 44 additions & 9 deletions crates/libfmp/src/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@ use url::Url;
use crate::{
Result, VERSION,
config::{Authentication, DEFAULT_BASE_URL, DEFAULT_PATH_PREFIX, RedirectPolicy},
endpoints::{DecodeFailure, QueryEncoder, ResponseMetadata},
endpoints::{
DecodeFailure, ExpectedContentType, QueryEncoder, ResponseMetadata, is_provider_message,
},
error::{ConfigurationErrorKind, Error, Redactor, SafeBody, SecretString},
transport::{HttpExecutor, PreparedRequest, ReqwestExecutor, TransportResponse},
};
Expand Down Expand Up @@ -476,31 +478,43 @@ impl Client {
return status_error(endpoint.id(), response, redactor);
}

let expected = endpoint.response().expected_content_type();
let Some(content_type) = response.headers().get(CONTENT_TYPE) else {
return Err(Error::decode(
Some(endpoint.id()),
Some(response.status()),
Some(safe_body(response.body(), redactor)),
Some(error_body(expected, response.body(), None, redactor)),
"successful response omitted its content type",
));
};
let Ok(content_type) = content_type.to_str() else {
return Err(Error::decode(
Some(endpoint.id()),
Some(response.status()),
Some(safe_body(response.body(), redactor)),
Some(error_body(expected, response.body(), None, redactor)),
"successful response used an invalid content type",
));
};
if !endpoint
.response()
.expected_content_type()
.matches(content_type)
{
if !expected.matches(content_type) && !expected.matches_signature(response.body()) {
if expected.is_binary()
&& ExpectedContentType::Json.matches(content_type)
&& is_provider_message(response.body())
{
return Err(Error::provider_message(
endpoint.id(),
response.status(),
Some(safe_body(response.body(), redactor)),
));
}
return Err(Error::decode(
Some(endpoint.id()),
Some(response.status()),
Some(safe_body(response.body(), redactor)),
Some(error_body(
expected,
response.body(),
Some(content_type),
redactor,
)),
"successful response used an unexpected content type",
));
}
Expand Down Expand Up @@ -1055,6 +1069,27 @@ fn safe_body(body: &[u8], redactor: &Redactor) -> SafeBody {
SafeBody::new(&String::from_utf8_lossy(body), redactor)
}

/// Builds the body of a decode error, summarizing it for binary endpoints so
/// raw payload bytes never reach a diagnostic (#411).
fn error_body(
expected: ExpectedContentType,
body: &[u8],
content_type: Option<&str>,
redactor: &Redactor,
) -> SafeBody {
if !expected.is_binary() {
return safe_body(body, redactor);
}
let summary = match content_type {
Some(content_type) => format!(
"<binary body omitted: {} bytes, content-type {content_type}>",
body.len()
),
None => format!("<binary body omitted: {} bytes>", body.len()),
};
SafeBody::new(&summary, redactor)
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down
32 changes: 31 additions & 1 deletion crates/libfmp/src/endpoints/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -330,8 +330,38 @@ impl ExpectedContentType {
.any(|expected| media_type.eq_ignore_ascii_case(expected)),
}
}

pub(crate) const fn is_binary(self) -> bool {
matches!(self, Self::Binary(_))
}

/// Reports whether `body` opens with the signature of one of the expected
/// binary formats, whatever its declared media type.
///
/// FMP has served real workbooks labeled `application/json` (#411). Only
/// the `Binary` arm sniffs; formats without a table entry never match.
pub(crate) fn matches_signature(self, body: &[u8]) -> bool {
let Self::Binary(expected) = self else {
return false;
};
expected.iter().any(|expected| {
BINARY_SIGNATURES.iter().any(|(media_type, magic)| {
expected.eq_ignore_ascii_case(media_type) && body.starts_with(magic)
})
})
}
}

const ZIP_MAGIC: &[u8] = b"PK\x03\x04";

const BINARY_SIGNATURES: &[(&str, &[u8])] = &[
(
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
ZIP_MAGIC,
),
("application/zip", ZIP_MAGIC),
];

fn valid_json_media_type(media_type: &str) -> bool {
let Some((type_name, subtype)) = media_type.split_once('/') else {
return false;
Expand Down Expand Up @@ -510,7 +540,7 @@ const ERROR_MESSAGE_MEMBER: &str = "Error Message";
/// One is a short plain-text line such as `Invalid name` that is not JSON at
/// all; the other is a JSON object whose only member is `"Error Message"`.
/// Both checks are bounded by size so ordinary payloads are not parsed twice.
fn is_provider_message(body: &[u8]) -> bool {
pub(crate) fn is_provider_message(body: &[u8]) -> bool {
let body = body.trim_ascii();
match body.first() {
None => false,
Expand Down
7 changes: 5 additions & 2 deletions crates/libfmp/src/endpoints/statements/reports.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,11 @@ use crate::{

/// Official XLSX MIME followed by the binary fallback accepted for proxy compatibility.
///
/// This is an SDK interoperability policy. The endpoint does not inspect ZIP
/// magic and rejects every other or missing response content type.
/// This is an SDK interoperability policy. A body under any other content
/// type is still accepted when it starts with the ZIP magic `PK\x03\x04`,
/// because FMP labels real workbooks `application/json` (#411); its content
/// type is kept as received. A missing content type, or any other body, is
/// rejected without echoing the body bytes.
pub const FINANCIAL_REPORTS_XLSX_CONTENT_TYPES: &[&str] = &[
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/octet-stream",
Expand Down
5 changes: 3 additions & 2 deletions crates/libfmp/tests/endpoints.rs
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,7 @@ async fn binary_response_equality_includes_bytes_and_exact_content_metadata() {
#[tokio::test]
async fn binary_contract_rejects_missing_and_wrong_content_types() {
const BINARY_TYPE: &str = "application/octet-stream";
const BINARY_BYTES: &[u8] = b"future-binary-response";
const BINARY_BYTES: &[u8] = b"future-binary-response\x00";
let endpoint: EndpointSpec<(), BinaryBody> = EndpointSpec::get_binary(
"binary-contract-test",
"future-download",
Expand Down Expand Up @@ -322,7 +322,8 @@ async fn unexpected_binary_content_has_bounded_token_safe_diagnostics() {
let diagnostic = format!("{error:?} {error}");

assert_eq!(error.category(), ErrorCategory::Decode);
assert!(safe_body.is_truncated());
assert!(!safe_body.is_truncated());
assert!(safe_body.as_str().starts_with("<binary body omitted: "));
assert!(safe_body.as_str().len() <= MAX_SAFE_BODY_BYTES);
assert!(!diagnostic.contains(AUTH_SECRET));
assert!(!diagnostic.contains(BODY_SECRET));
Expand Down
104 changes: 103 additions & 1 deletion crates/libfmp/tests/statements_report_endpoints.rs
Original file line number Diff line number Diff line change
Expand Up @@ -349,9 +349,111 @@ async fn rejected_xlsx_diagnostics_are_bounded_and_hide_tokens_and_links() {
let diagnostic = format!("{error:?} {error}");

assert_eq!(error.category(), ErrorCategory::Decode);
assert!(safe_body.is_truncated());
assert!(!safe_body.is_truncated());
assert!(safe_body.as_str().starts_with("<binary body omitted: "));
assert!(safe_body.as_str().len() <= MAX_SAFE_BODY_BYTES);
assert!(!safe_body.as_str().contains(BODY_SECRET));
assert!(!diagnostic.contains(AUTH_SECRET));
assert!(!diagnostic.contains(BODY_SECRET));
}

const LABELED_JSON: &str = "application/json; charset=utf-8";
const ZIP_WORKBOOK: &[u8] = b"PK\x03\x04workbook-entries-private-bytes";

async fn xlsx_call(response: TransportResponse) -> libfmp::Result<BinaryResponse> {
let client = Client::builder()
.authentication(Authentication::fmp_header("secret"))
.executor(Arc::new(FixtureExecutor::new([response])))
.build()
.unwrap();
client
.financial_reports_xlsx(FinancialReportsXlsxQuery::new(
Ticker::new("AAPL").unwrap(),
Year(2024),
FiscalPeriod::FullYear,
))
.await
}

#[tokio::test]
async fn xlsx_accepts_a_zip_workbook_labeled_as_json_and_keeps_its_content_type() {
const DISPOSITION: &str = "attachment; filename = AAPL_2024_FY_.xlsx";
let official = xlsx_call(binary_fixture(Some(OFFICIAL_XLSX), None, ZIP_WORKBOOK))
.await
.unwrap();
let labeled = xlsx_call(binary_fixture(
Some(LABELED_JSON),
Some(DISPOSITION),
ZIP_WORKBOOK,
))
.await
.unwrap();

assert_eq!(official.as_bytes(), ZIP_WORKBOOK);
assert_eq!(official.content_type(), OFFICIAL_XLSX);
assert_eq!(labeled.as_bytes(), ZIP_WORKBOOK);
assert_eq!(labeled.content_type(), LABELED_JSON);
assert_eq!(labeled.content_disposition(), Some(DISPOSITION));
}

#[tokio::test]
async fn xlsx_labeled_as_json_keeps_the_provider_message_check() {
const MESSAGE: &[u8] = br#"{"Error Message":"Limit reached for xlsx"}"#;
let error = xlsx_call(binary_fixture(Some(LABELED_JSON), None, MESSAGE))
.await
.unwrap_err();

assert_eq!(error.category(), ErrorCategory::Status);
assert_eq!(error.status_code(), Some(200));
assert_eq!(error.endpoint(), Some("financial-reports-xlsx"));
assert_eq!(
error.body().unwrap().as_str(),
r#"{"Error Message":"Limit reached for xlsx"}"#
);
}

#[tokio::test]
async fn xlsx_rejects_other_bodies_without_echoing_their_bytes() {
const GARBAGE: &[u8] = b"\x00\x01garbage-private-payload\xff";
let error = xlsx_call(binary_fixture(Some(LABELED_JSON), None, GARBAGE))
.await
.unwrap_err();
let diagnostic = format!("{error:?} {error}");

assert_eq!(error.category(), ErrorCategory::Decode);
assert_eq!(error.status_code(), Some(200));
assert_eq!(
error.body().unwrap().as_str(),
format!(
"<binary body omitted: {} bytes, content-type {LABELED_JSON}>",
GARBAGE.len()
)
);
assert!(!diagnostic.contains("garbage-private-payload"));
}

#[tokio::test]
async fn json_endpoints_never_sniff_workbook_signatures() {
let executor = Arc::new(FixtureExecutor::new([
binary_fixture(Some(OFFICIAL_XLSX), None, ZIP_WORKBOOK),
binary_fixture(Some("application/json"), None, ZIP_WORKBOOK),
]));
let client = Client::builder()
.authentication(Authentication::fmp_header("secret"))
.executor(executor)
.build()
.unwrap();
let query = || {
FinancialReportsJsonQuery::new(
Ticker::new("AAPL").unwrap(),
Year(2024),
FiscalPeriod::FullYear,
)
};

for _ in 0..2 {
let error = client.financial_reports_json(query()).await.unwrap_err();
assert_eq!(error.category(), ErrorCategory::Decode);
assert_eq!(error.endpoint(), Some("financial-reports-json"));
}
}
4 changes: 3 additions & 1 deletion docs/adr/0002-shared-wire-types-and-python-conversions.md
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,9 @@ contents, or content-type parameters.
Each endpoint owns its accepted MIME allow-list because the provider source does
not establish one global binary MIME policy. The client validates `Content-Type`
against that endpoint contract before constructing `BinaryResponse`; optional
`Content-Disposition` is retained only when it is valid header text.
`Content-Disposition` is retained only when it is valid header text. Since #411
a body with the magic bytes of an expected format is also accepted under a
mismatched content type (ADR 0008).

`PartialEq` and `Eq` compare bytes, exact content type, and optional content
disposition. Metadata is part of binary response identity rather than incidental
Expand Down
11 changes: 11 additions & 0 deletions docs/adr/0008-as-reported-and-financial-reports-api-names.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,17 @@ though the endpoint is documented as an XLSX download. The SDK therefore uses
`application/octet-stream` for binary-compatible proxies; it does not model
the copied JSON example as the XLSX response contract.

Update 2026-10-04 (#411): FMP now serves the real workbook with
`Content-Type: application/json; charset=utf-8`. The SDK (Rust and Go) also
accepts a successful body that starts with the magic bytes of an expected
format (XLSX and the ZIP family: `PK\x03\x04`), whatever its declared media
type; the content type is kept exactly as received. `Content-Disposition` is
not used for the decision. A JSON-labeled body that is an FMP provider
message still yields the provider-message error. Every other error on a
binary endpoint summarises the body (`<binary body omitted: N bytes, ...>`)
instead of echoing its bytes. `application/octet-stream` has no signature and
is never sniffed.

`FinancialReportDate` links use `SecretUrl` so embedded API keys remain
redacted in debug and display surfaces; implicit serialization is disallowed.
Revenue segmentation uses the shared dynamic row type and the closed documented query value
Expand Down
45 changes: 40 additions & 5 deletions sdk/go/binary.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package fmp

import (
"bytes"
"context"
"fmt"
"strings"
Expand Down Expand Up @@ -32,7 +33,10 @@ func (p BinaryPayload) Format(f fmt.State, _ rune) {

// getBinary issues one GET for a logical endpoint and returns the raw body
// when the response carries one of the expected media types, compared
// case-insensitively without parameters. endpointID is the registry id
// case-insensitively without parameters, or when the body opens with the
// signature of one of them (FMP has labeled real workbooks as JSON, #411).
// A JSON-labeled provider message keeps its provider-message error; every
// other error summarizes the body instead of echoing its bytes. endpointID is the registry id
// carried by errors, never a URL. The redirect, timeout, body cap, and
// no-retry rules of getJSON apply unchanged.
func (c *Client) getBinary(ctx context.Context, endpointID, relativePath string, query []queryParam,
Expand All @@ -57,13 +61,16 @@ func (c *Client) getBinary(ctx context.Context, endpointID, relativePath string,
contentType := resp.header.Get("Content-Type")
switch {
case contentType == "":
return BinaryPayload{}, decodeError(endpointID, resp.status, c.safeBody(resp.body),
return BinaryPayload{}, decodeError(endpointID, resp.status, c.binaryErrorBody(resp.body, ""),
"successful response omitted its content type", nil)
case !visibleHeaderText(contentType):
return BinaryPayload{}, decodeError(endpointID, resp.status, c.safeBody(resp.body),
return BinaryPayload{}, decodeError(endpointID, resp.status, c.binaryErrorBody(resp.body, ""),
"successful response used an invalid content type", nil)
case !matchesMediaType(contentType, expectedContentTypes):
return BinaryPayload{}, decodeError(endpointID, resp.status, c.safeBody(resp.body),
case !matchesMediaType(contentType, expectedContentTypes) && !matchesSignature(resp.body, expectedContentTypes):
if validJSONMediaType(contentType) && isProviderMessage(resp.body) {
return BinaryPayload{}, providerMessageError(endpointID, resp.status, c.safeBody(resp.body))
}
return BinaryPayload{}, decodeError(endpointID, resp.status, c.binaryErrorBody(resp.body, contentType),
"successful response used an unexpected content type", nil)
}
disposition := resp.header.Get("Content-Disposition")
Expand All @@ -87,3 +94,31 @@ func matchesMediaType(contentType string, expected []string) bool {
}
return false
}

var zipMagic = []byte("PK\x03\x04")

// binarySignatures maps binary media types to the magic bytes their bodies
// start with. Types without an entry are never sniffed.
var binarySignatures = map[string][]byte{
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": zipMagic,
"application/zip": zipMagic,
}

func matchesSignature(body []byte, expected []string) bool {
for _, candidate := range expected {
if magic, ok := binarySignatures[strings.ToLower(candidate)]; ok && bytes.HasPrefix(body, magic) {
return true
}
}
return false
}

// binaryErrorBody summarizes a rejected binary body without its bytes.
func (c *Client) binaryErrorBody(body []byte, contentType string) *SafeBody {
summary := fmt.Sprintf("<binary body omitted: %d bytes>", len(body))
if contentType != "" {
summary = fmt.Sprintf("<binary body omitted: %d bytes, content-type %s>", len(body), contentType)
}
safe := NewSafeBody(summary, c.redactor)
return &safe
}
Loading
Loading