From ea6f8dd15b78be7a096bfdf6dba0238650e722d1 Mon Sep 17 00:00:00 2001 From: benbenbang Date: Wed, 30 Sep 2026 20:51:34 +0800 Subject: [PATCH 1/2] feat(search): make omittable ExchangeVariant and screener members optional Members that FMP can send as null or omit on ExchangeVariant and CompanyScreenerResult previously failed decoding a whole page when any single value was null. These members are now optional so a null value decodes to None instead of rejecting the response. **Rust response models:** * CompanyScreenerResult now types beta, price, lastAnnualDividend, country, and isFund as Option, decoding null to None and re-encoding None as null. * ExchangeVariant now types 15 members (price, range, changes, cik, isin, cusip, website, ceo, fullTimeEmployees, phone, address, city, state, zip, dcfDiff) as Option; an empty cusip also decodes to None. **Go SDK:** * CompanyScreenerResult and ExchangeVariant use pointer types for the same members, decoding null to nil and re-encoding nil as null. **Python bindings:** * The pyo3 models and generated .pyi stubs expose the same members as Optional and handle None. Because these members were previously required, code that read them as plain values must now handle the absent case. BREAKING CHANGE: ExchangeVariant and CompanyScreenerResult members FMP can omit are now optional (Option / pointer / Optional); handle None. --- .../python/fmp/_native/screener/__init__.pyi | 20 +- .../python/fmp/_native/search/__init__.pyi | 60 ++-- crates/fmp-py/src/models/screener.rs | 22 +- crates/fmp-py/src/models/search.rs | 66 ++-- crates/fmp-py/tests/test_screener.py | 31 +- crates/fmp-py/tests/test_search_a.py | 18 + crates/fmp-py/tests/typecheck_contract.py | 7 +- crates/libfmp/src/codecs.rs | 1 - crates/libfmp/src/responses/screener.rs | 29 +- crates/libfmp/src/responses/search.rs | 57 +++- crates/libfmp/tests/decode_path.rs | 36 +- crates/libfmp/tests/screener_responses.rs | 47 ++- crates/libfmp/tests/search_responses.rs | 95 +++++- sdk/go/decode_path_test.go | 16 +- sdk/go/screener_models.go | 120 ++++--- sdk/go/screener_parity_test.go | 58 +++- sdk/go/screener_test.go | 2 +- sdk/go/search_models.go | 308 ++++++++++++------ sdk/go/search_parity_test.go | 55 +++- 19 files changed, 740 insertions(+), 308 deletions(-) diff --git a/crates/fmp-py/python/fmp/_native/screener/__init__.pyi b/crates/fmp-py/python/fmp/_native/screener/__init__.pyi index 5aec46d6..ec5ba1fe 100644 --- a/crates/fmp-py/python/fmp/_native/screener/__init__.pyi +++ b/crates/fmp-py/python/fmp/_native/screener/__init__.pyi @@ -41,11 +41,11 @@ class CompanyScreenerResult: @property def industry(self) -> builtins.str: ... @property - def beta(self) -> builtins.float: ... + def beta(self) -> typing.Optional[builtins.float]: ... @property - def price(self) -> builtins.float: ... + def price(self) -> typing.Optional[builtins.float]: ... @property - def last_annual_dividend(self) -> builtins.float: ... + def last_annual_dividend(self) -> typing.Optional[builtins.float]: ... @property def volume(self) -> builtins.float: ... @property @@ -53,11 +53,11 @@ class CompanyScreenerResult: @property def exchange_short_name(self) -> builtins.str: ... @property - def country(self) -> builtins.str: ... + def country(self) -> typing.Optional[builtins.str]: ... @property def is_etf(self) -> builtins.bool: ... @property - def is_fund(self) -> builtins.bool: ... + def is_fund(self) -> typing.Optional[builtins.bool]: ... @property def is_actively_trading(self) -> builtins.bool: ... def __eq__(self, other: builtins.object, /) -> builtins.bool: ... @@ -69,15 +69,15 @@ class CompanyScreenerResult: market_cap: builtins.float, sector: builtins.str, industry: builtins.str, - beta: builtins.float, - price: builtins.float, - last_annual_dividend: builtins.float, + beta: typing.Optional[builtins.float], + price: typing.Optional[builtins.float], + last_annual_dividend: typing.Optional[builtins.float], volume: builtins.float, exchange: builtins.str, exchange_short_name: builtins.str, - country: builtins.str, + country: typing.Optional[builtins.str], is_etf: builtins.bool, - is_fund: builtins.bool, + is_fund: typing.Optional[builtins.bool], is_actively_trading: builtins.bool, ) -> CompanyScreenerResult: ... def __getnewargs_ex__(self) -> tuple[tuple, dict]: ... diff --git a/crates/fmp-py/python/fmp/_native/search/__init__.pyi b/crates/fmp-py/python/fmp/_native/search/__init__.pyi index 2c498206..db96cf2f 100644 --- a/crates/fmp-py/python/fmp/_native/search/__init__.pyi +++ b/crates/fmp-py/python/fmp/_native/search/__init__.pyi @@ -140,7 +140,7 @@ class ExchangeVariant: @property def symbol(self) -> builtins.str: ... @property - def price(self) -> builtins.float: ... + def price(self) -> typing.Optional[builtins.float]: ... @property def beta(self) -> builtins.float: ... @property @@ -150,19 +150,19 @@ class ExchangeVariant: @property def last_div(self) -> builtins.float: ... @property - def range(self) -> builtins.str: ... + def range(self) -> typing.Optional[builtins.str]: ... @property - def changes(self) -> builtins.float: ... + def changes(self) -> typing.Optional[builtins.float]: ... @property def company_name(self) -> builtins.str: ... @property def currency(self) -> builtins.str: ... @property - def cik(self) -> builtins.str: ... + def cik(self) -> typing.Optional[builtins.str]: ... @property - def isin(self) -> builtins.str: ... + def isin(self) -> typing.Optional[builtins.str]: ... @property - def cusip(self) -> builtins.str: ... + def cusip(self) -> typing.Optional[builtins.str]: ... @property def exchange(self) -> builtins.str: ... @property @@ -170,29 +170,29 @@ class ExchangeVariant: @property def industry(self) -> builtins.str: ... @property - def website(self) -> builtins.str: ... + def website(self) -> typing.Optional[builtins.str]: ... @property def description(self) -> builtins.str: ... @property - def ceo(self) -> builtins.str: ... + def ceo(self) -> typing.Optional[builtins.str]: ... @property def sector(self) -> builtins.str: ... @property def country(self) -> builtins.str: ... @property - def full_time_employees(self) -> builtins.str: ... + def full_time_employees(self) -> typing.Optional[builtins.str]: ... @property - def phone(self) -> builtins.str: ... + def phone(self) -> typing.Optional[builtins.str]: ... @property - def address(self) -> builtins.str: ... + def address(self) -> typing.Optional[builtins.str]: ... @property - def city(self) -> builtins.str: ... + def city(self) -> typing.Optional[builtins.str]: ... @property - def state(self) -> builtins.str: ... + def state(self) -> typing.Optional[builtins.str]: ... @property - def zip(self) -> builtins.str: ... + def zip(self) -> typing.Optional[builtins.str]: ... @property - def dcf_diff(self) -> builtins.float: ... + def dcf_diff(self) -> typing.Optional[builtins.float]: ... @property def dcf(self) -> builtins.float: ... @property @@ -214,33 +214,33 @@ class ExchangeVariant: cls, *, symbol: builtins.str, - price: builtins.float, + price: typing.Optional[builtins.float], beta: builtins.float, vol_avg: builtins.float, market_cap: builtins.float, last_div: builtins.float, - range: builtins.str, - changes: builtins.float, + range: typing.Optional[builtins.str], + changes: typing.Optional[builtins.float], company_name: builtins.str, currency: builtins.str, - cik: builtins.str, - isin: builtins.str, - cusip: builtins.str, + cik: typing.Optional[builtins.str], + isin: typing.Optional[builtins.str], + cusip: typing.Optional[builtins.str], exchange: builtins.str, exchange_short_name: builtins.str, industry: builtins.str, - website: builtins.str, + website: typing.Optional[builtins.str], description: builtins.str, - ceo: builtins.str, + ceo: typing.Optional[builtins.str], sector: builtins.str, country: builtins.str, - full_time_employees: builtins.str, - phone: builtins.str, - address: builtins.str, - city: builtins.str, - state: builtins.str, - zip: builtins.str, - dcf_diff: builtins.float, + full_time_employees: typing.Optional[builtins.str], + phone: typing.Optional[builtins.str], + address: typing.Optional[builtins.str], + city: typing.Optional[builtins.str], + state: typing.Optional[builtins.str], + zip: typing.Optional[builtins.str], + dcf_diff: typing.Optional[builtins.float], dcf: builtins.float, image: builtins.str, ipo_date: datetime.date, diff --git a/crates/fmp-py/src/models/screener.rs b/crates/fmp-py/src/models/screener.rs index 68027b77..00e6bad4 100644 --- a/crates/fmp-py/src/models/screener.rs +++ b/crates/fmp-py/src/models/screener.rs @@ -20,11 +20,11 @@ pub(crate) struct CompanyScreenerResult { #[pyo3(get)] pub industry: String, #[pyo3(get)] - pub beta: f64, + pub beta: Option, #[pyo3(get)] - pub price: f64, + pub price: Option, #[pyo3(get)] - pub last_annual_dividend: f64, + pub last_annual_dividend: Option, #[pyo3(get)] pub volume: f64, #[pyo3(get)] @@ -32,11 +32,11 @@ pub(crate) struct CompanyScreenerResult { #[pyo3(get)] pub exchange_short_name: String, #[pyo3(get)] - pub country: String, + pub country: Option, #[pyo3(get)] pub is_etf: bool, #[pyo3(get)] - pub is_fund: bool, + pub is_fund: Option, #[pyo3(get)] pub is_actively_trading: bool, } @@ -54,15 +54,15 @@ impl CompanyScreenerResult { market_cap: f64, sector: String, industry: String, - beta: f64, - price: f64, - last_annual_dividend: f64, + beta: Option, + price: Option, + last_annual_dividend: Option, volume: f64, exchange: String, exchange_short_name: String, - country: String, + country: Option, is_etf: bool, - is_fund: bool, + is_fund: Option, is_actively_trading: bool, ) -> Self { Self { @@ -266,7 +266,7 @@ impl From for CompanyScreene volume: value.volume, exchange: value.exchange, exchange_short_name: value.exchange_short_name.into_inner(), - country: value.country.into_inner(), + country: value.country.map(|value| value.into_inner()), is_etf: value.is_etf, is_fund: value.is_fund, is_actively_trading: value.is_actively_trading, diff --git a/crates/fmp-py/src/models/search.rs b/crates/fmp-py/src/models/search.rs index bf63dbe1..8d12bcfb 100644 --- a/crates/fmp-py/src/models/search.rs +++ b/crates/fmp-py/src/models/search.rs @@ -659,7 +659,7 @@ pub(crate) struct ExchangeVariant { #[pyo3(get)] pub symbol: String, #[pyo3(get)] - pub price: f64, + pub price: Option, #[pyo3(get)] pub beta: f64, #[pyo3(get)] @@ -669,19 +669,19 @@ pub(crate) struct ExchangeVariant { #[pyo3(get)] pub last_div: f64, #[pyo3(get)] - pub range: String, + pub range: Option, #[pyo3(get)] - pub changes: f64, + pub changes: Option, #[pyo3(get)] pub company_name: String, #[pyo3(get)] pub currency: String, #[pyo3(get)] - pub cik: String, + pub cik: Option, #[pyo3(get)] - pub isin: String, + pub isin: Option, #[pyo3(get)] - pub cusip: String, + pub cusip: Option, #[pyo3(get)] pub exchange: String, #[pyo3(get)] @@ -689,29 +689,29 @@ pub(crate) struct ExchangeVariant { #[pyo3(get)] pub industry: String, #[pyo3(get)] - pub website: String, + pub website: Option, #[pyo3(get)] pub description: String, #[pyo3(get)] - pub ceo: String, + pub ceo: Option, #[pyo3(get)] pub sector: String, #[pyo3(get)] pub country: String, #[pyo3(get)] - pub full_time_employees: String, + pub full_time_employees: Option, #[pyo3(get)] - pub phone: String, + pub phone: Option, #[pyo3(get)] - pub address: String, + pub address: Option, #[pyo3(get)] - pub city: String, + pub city: Option, #[pyo3(get)] - pub state: String, + pub state: Option, #[pyo3(get)] - pub zip: String, + pub zip: Option, #[pyo3(get)] - pub dcf_diff: f64, + pub dcf_diff: Option, #[pyo3(get)] pub dcf: f64, #[pyo3(get)] @@ -739,33 +739,33 @@ impl ExchangeVariant { #[pyo3(signature = (*, symbol, price, beta, vol_avg, market_cap, last_div, range, changes, company_name, currency, cik, isin, cusip, exchange, exchange_short_name, industry, website, description, ceo, sector, country, full_time_employees, phone, address, city, state, zip, dcf_diff, dcf, image, ipo_date, default_image, is_etf, is_actively_trading, is_adr, is_fund))] fn new( symbol: String, - price: f64, + price: Option, beta: f64, vol_avg: f64, market_cap: f64, last_div: f64, - range: String, - changes: f64, + range: Option, + changes: Option, company_name: String, currency: String, - cik: String, - isin: String, - cusip: String, + cik: Option, + isin: Option, + cusip: Option, exchange: String, exchange_short_name: String, industry: String, - website: String, + website: Option, description: String, - ceo: String, + ceo: Option, sector: String, country: String, - full_time_employees: String, - phone: String, - address: String, - city: String, - state: String, - zip: String, - dcf_diff: f64, + full_time_employees: Option, + phone: Option, + address: Option, + city: Option, + state: Option, + zip: Option, + dcf_diff: Option, dcf: f64, image: String, ipo_date: ::chrono::NaiveDate, @@ -1155,9 +1155,9 @@ impl From for ExchangeVariant { changes: value.changes, company_name: value.company_name, currency: value.currency.into_inner(), - cik: value.cik.into_inner(), - isin: value.isin.into_inner(), - cusip: value.cusip.into_inner(), + cik: value.cik.map(|value| value.into_inner()), + isin: value.isin.map(|value| value.into_inner()), + cusip: value.cusip.map(|value| value.into_inner()), exchange: value.exchange, exchange_short_name: value.exchange_short_name.into_inner(), industry: value.industry, diff --git a/crates/fmp-py/tests/test_screener.py b/crates/fmp-py/tests/test_screener.py index 528a24e7..a8e5e3fc 100644 --- a/crates/fmp-py/tests/test_screener.py +++ b/crates/fmp-py/tests/test_screener.py @@ -219,18 +219,39 @@ def test_decode_error_for_a_non_json_body(client: Any, fixture_server: FixtureSe assert (caught.value.decode_kind, caught.value.decode_path) == ("syntax", None) +def test_companies_decodes_a_page_with_a_null_beta_row(client: Any, fixture_server: FixtureServer) -> None: + """Issue #368: a ``null`` beta no longer fails the page; the row's nullable members read ``None``.""" + rows = load_fixture("company_screener_null_beta_synthetic.json") + for member in ("price", "lastAnnualDividend", "country", "isFund"): + rows[36][member] = None + fixture_server.route("/company-screener", rows) + decoded = client.screener.companies() + + assert len(decoded) == 40 + assert decoded[37].beta is None + assert decoded[35].beta == pytest.approx(1.097) + assert (decoded[36].price, decoded[36].last_annual_dividend, decoded[36].country, decoded[36].is_fund) == ( + None, + None, + None, + None, + ) + + def test_decode_error_names_the_null_member_row_and_member( client: Any, fixture_server: FixtureServer, errors: SimpleNamespace ) -> None: """Issue #366: a ``null`` required member names its row index and member.""" - fixture_server.route("/company-screener", load_fixture("company_screener_null_beta_synthetic.json")) + rows = load_fixture("company_screener_null_beta_synthetic.json") + rows[37]["companyName"] = None + fixture_server.route("/company-screener", rows) with pytest.raises(errors.FmpDecodeError) as caught: client.screener.companies() - assert caught.value.decode_path == "[37].beta" + assert caught.value.decode_path == "[37].companyName" assert caught.value.decode_kind == "null" assert str(caught.value).startswith( - "successful response could not be decoded: null value at [37].beta (endpoint: company-screener): " + "successful response could not be decoded: null value at [37].companyName (endpoint: company-screener): " ) @@ -240,13 +261,13 @@ def test_decode_error_never_carries_a_string_member_value( """Issue #366: the offending string value reaches neither the message nor an attribute.""" sentinel = "SENTINEL-beta-text" rows = load_fixture("company_screener_null_beta_synthetic.json") - rows[37]["beta"] = sentinel + rows[37]["volume"] = sentinel fixture_server.route("/company-screener", rows) with pytest.raises(errors.FmpDecodeError) as caught: client.screener.companies() error = caught.value - assert error.decode_path == "[37].beta" + assert error.decode_path == "[37].volume" assert error.decode_kind == "wrong_type" assert error.body_truncated is True assert sentinel not in str(error) diff --git a/crates/fmp-py/tests/test_search_a.py b/crates/fmp-py/tests/test_search_a.py index 8d42222f..8fb040ac 100644 --- a/crates/fmp-py/tests/test_search_a.py +++ b/crates/fmp-py/tests/test_search_a.py @@ -237,6 +237,24 @@ def test_exchange_variants_encodes_the_caret(client: Any, fixture_server: Fixtur assert row.default_image is False +@pytest.mark.parametrize("cusip", [None, ""]) +def test_exchange_variants_decodes_null_members_and_empty_cusip_to_none( + client: Any, fixture_server: FixtureServer, cusip: str | None +) -> None: + """Issue #368: the 15 members FMP can send as ``null`` read ``None``; an empty ``cusip`` does too.""" + nullable = ("price", "range", "changes", "cik", "isin", "website", "ceo", "fullTimeEmployees", "phone") + nullable += ("address", "city", "state", "zip", "dcfDiff") + rows = load_fixture("search_exchange_variants.json") + rows[0].update(dict.fromkeys(nullable), cusip=cusip) + fixture_server.route("/search-exchange-variants", rows) + row = client.search.exchange_variants("AAPL")[0] + + attributes = ("price", "range", "changes", "cik", "isin", "cusip", "website", "ceo", "full_time_employees") + attributes += ("phone", "address", "city", "state", "zip", "dcf_diff") + assert {name: getattr(row, name) for name in attributes} == dict.fromkeys(attributes) + assert row.symbol == "AAPL" + + @pytest.mark.parametrize( ("method", "argument", "path"), [ diff --git a/crates/fmp-py/tests/typecheck_contract.py b/crates/fmp-py/tests/typecheck_contract.py index 7f7db1d3..3b2db0f4 100644 --- a/crates/fmp-py/tests/typecheck_contract.py +++ b/crates/fmp-py/tests/typecheck_contract.py @@ -318,8 +318,8 @@ def check_screener_contract(client: FmpClient) -> None: limit=100, ) market_cap: float = filtered[0].market_cap - beta: float = everything[0].beta - is_fund: bool = filtered[0].is_fund + beta: float | None = everything[0].beta + is_fund: bool | None = filtered[0].is_fund _ = (market_cap, beta, is_fund) @@ -460,7 +460,8 @@ def check_search_contract(client: FmpClient) -> None: ipo_date: datetime.date = variants[0].ipo_date market_cap: float = variants[0].market_cap is_etf: bool = variants[0].is_etf - _ = (exchange_full_name, cik, ipo_date, market_cap, is_etf) + cusip: str | None = variants[0].cusip + _ = (exchange_full_name, cik, ipo_date, market_cap, is_etf, cusip) def check_insider_trading_contract(client: FmpClient) -> None: diff --git a/crates/libfmp/src/codecs.rs b/crates/libfmp/src/codecs.rs index a7e1398f..4ac24f6a 100644 --- a/crates/libfmp/src/codecs.rs +++ b/crates/libfmp/src/codecs.rs @@ -723,7 +723,6 @@ pub(crate) mod empty_or_null { use crate::types::StringValueError; - #[cfg_attr(not(test), allow(dead_code))] pub(crate) fn deserialize<'de, D, T>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, diff --git a/crates/libfmp/src/responses/screener.rs b/crates/libfmp/src/responses/screener.rs index 004d8711..8d8eda44 100644 --- a/crates/libfmp/src/responses/screener.rs +++ b/crates/libfmp/src/responses/screener.rs @@ -2,13 +2,21 @@ //! //! The Python binding exposes this model under `fmp.screener`. -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use crate::types::{ CountryCode, ExchangeCode, Industry, MarketCapitalization, PerShareAmount, Price, Ratio, Sector, Ticker, Volume, }; +fn required_option<'de, D, T>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, + T: Deserialize<'de>, +{ + Option::::deserialize(deserializer) +} + /// One worldwide company returned by the stock screener. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[non_exhaustive] @@ -20,19 +28,24 @@ pub struct CompanyScreenerResult { pub market_cap: MarketCapitalization, pub sector: Sector, pub industry: Industry, - #[serde(serialize_with = "crate::codecs::integral_f64::serialize")] - pub beta: Ratio, - pub price: Price, - #[serde(serialize_with = "crate::codecs::integral_f64::serialize")] - pub last_annual_dividend: PerShareAmount, + #[serde(deserialize_with = "required_option")] + #[serde(serialize_with = "crate::codecs::integral_f64::serialize_option")] + pub beta: Option, + #[serde(deserialize_with = "required_option")] + pub price: Option, + #[serde(deserialize_with = "required_option")] + #[serde(serialize_with = "crate::codecs::integral_f64::serialize_option")] + pub last_annual_dividend: Option, #[serde(serialize_with = "crate::codecs::integral_f64::serialize")] pub volume: Volume, /// The provider's full exchange name. pub exchange: String, /// The provider's abbreviated exchange code. pub exchange_short_name: ExchangeCode, - pub country: CountryCode, + #[serde(deserialize_with = "required_option")] + pub country: Option, pub is_etf: bool, - pub is_fund: bool, + #[serde(deserialize_with = "required_option")] + pub is_fund: Option, pub is_actively_trading: bool, } diff --git a/crates/libfmp/src/responses/search.rs b/crates/libfmp/src/responses/search.rs index 7e888dcd..3388502e 100644 --- a/crates/libfmp/src/responses/search.rs +++ b/crates/libfmp/src/responses/search.rs @@ -2,13 +2,21 @@ //! //! The Python binding exposes these models under `fmp.search`. -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use crate::types::{ Change, Cik, CountryCode, CurrencyCode, Cusip, Date, ExchangeCode, Isin, MarketCapitalization, PerShareAmount, Price, Ratio, Ticker, Volume, }; +fn required_option<'de, D, T>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, + T: Deserialize<'de>, +{ + Option::::deserialize(deserializer) +} + /// A company or instrument returned by symbol search. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[non_exhaustive] @@ -76,7 +84,8 @@ pub struct IsinSearchResult { #[serde(rename_all = "camelCase")] pub struct ExchangeVariant { pub symbol: Ticker, - pub price: Price, + #[serde(deserialize_with = "required_option")] + pub price: Option, #[serde(serialize_with = "crate::codecs::integral_f64::serialize")] pub beta: Ratio, #[serde(serialize_with = "crate::codecs::integral_f64::serialize")] @@ -86,31 +95,45 @@ pub struct ExchangeVariant { pub market_cap: MarketCapitalization, #[serde(serialize_with = "crate::codecs::integral_f64::serialize")] pub last_div: PerShareAmount, - pub range: String, - pub changes: Change, + #[serde(deserialize_with = "required_option")] + pub range: Option, + #[serde(deserialize_with = "required_option")] + pub changes: Option, pub company_name: String, pub currency: CurrencyCode, - pub cik: Cik, - pub isin: Isin, - pub cusip: Cusip, + #[serde(deserialize_with = "required_option")] + pub cik: Option, + #[serde(deserialize_with = "required_option")] + pub isin: Option, + #[serde(deserialize_with = "crate::codecs::empty_or_null::deserialize")] + pub cusip: Option, /// The provider's `exchange` field is the full exchange name here. pub exchange: String, /// The provider's `exchangeShortName` field is the exchange code here. pub exchange_short_name: ExchangeCode, pub industry: String, - pub website: String, + #[serde(deserialize_with = "required_option")] + pub website: Option, pub description: String, - pub ceo: String, + #[serde(deserialize_with = "required_option")] + pub ceo: Option, pub sector: String, pub country: CountryCode, - pub full_time_employees: String, - pub phone: String, - pub address: String, - pub city: String, - pub state: String, - pub zip: String, - #[serde(serialize_with = "crate::codecs::integral_f64::serialize")] - pub dcf_diff: PerShareAmount, + #[serde(deserialize_with = "required_option")] + pub full_time_employees: Option, + #[serde(deserialize_with = "required_option")] + pub phone: Option, + #[serde(deserialize_with = "required_option")] + pub address: Option, + #[serde(deserialize_with = "required_option")] + pub city: Option, + #[serde(deserialize_with = "required_option")] + pub state: Option, + #[serde(deserialize_with = "required_option")] + pub zip: Option, + #[serde(deserialize_with = "required_option")] + #[serde(serialize_with = "crate::codecs::integral_f64::serialize_option")] + pub dcf_diff: Option, #[serde(serialize_with = "crate::codecs::integral_f64::serialize")] pub dcf: PerShareAmount, pub image: String, diff --git a/crates/libfmp/tests/decode_path.rs b/crates/libfmp/tests/decode_path.rs index 5593b43e..2bdd1501 100644 --- a/crates/libfmp/tests/decode_path.rs +++ b/crates/libfmp/tests/decode_path.rs @@ -43,14 +43,38 @@ async fn decode_error(response: TransportResponse) -> libfmp::Error { error } +#[tokio::test] +async fn nullable_beta_row_no_longer_fails_the_page() { + let executor = Arc::new(FixtureExecutor::new([json_fixture(NULL_BETA)])); + let client = Client::builder() + .base_url("https://proxy.example/router") + .path_prefix("stable") + .authentication(Authentication::None) + .executor(executor) + .build() + .unwrap(); + let rows = client + .company_screener(CompanyScreenerQuery::new()) + .await + .unwrap(); + + assert_eq!(rows.len(), 40); + assert_eq!(rows[37].beta, None); +} + #[tokio::test] async fn null_member_names_the_row_index_and_member() { - let error = decode_error(json_fixture(NULL_BETA)).await; + let body = String::from_utf8(NULL_BETA.to_vec()).unwrap().replacen( + "\"companyName\": \"Synthetic Row 37\"", + "\"companyName\": null", + 1, + ); + let error = decode_error(json_body(body)).await; - assert_eq!(error.decode_path(), Some("[37].beta")); + assert_eq!(error.decode_path(), Some("[37].companyName")); assert_eq!(error.decode_kind(), Some(DecodeErrorKind::Null)); assert!(error.to_string().starts_with( - "successful response could not be decoded: null value at [37].beta \ + "successful response could not be decoded: null value at [37].companyName \ (endpoint: company-screener): " )); } @@ -58,13 +82,13 @@ async fn null_member_names_the_row_index_and_member() { #[tokio::test] async fn string_valued_member_never_reaches_the_error_text() { let body = String::from_utf8(NULL_BETA.to_vec()).unwrap().replacen( - "\"beta\": null", - &format!("\"beta\": \"{SENTINEL}\""), + "\"symbol\": \"R37\",\n \"volume\": 29909012", + &format!("\"symbol\": \"R37\",\n \"volume\": \"{SENTINEL}\""), 1, ); let error = decode_error(json_body(body)).await; - assert_eq!(error.decode_path(), Some("[37].beta")); + assert_eq!(error.decode_path(), Some("[37].volume")); assert_eq!(error.decode_kind(), Some(DecodeErrorKind::WrongType)); assert!(error.body().unwrap().is_truncated()); assert!(!error.to_string().contains(SENTINEL)); diff --git a/crates/libfmp/tests/screener_responses.rs b/crates/libfmp/tests/screener_responses.rs index c2a4e9ca..31474804 100644 --- a/crates/libfmp/tests/screener_responses.rs +++ b/crates/libfmp/tests/screener_responses.rs @@ -12,15 +12,15 @@ fn documented_company_screener_entry_decodes_every_exact_field() { assert_eq!(row.market_cap, 4_885_602_246_714.0); assert_eq!(row.sector.as_str(), "Technology"); assert_eq!(row.industry.as_str(), "Consumer Electronics"); - assert_eq!(row.beta, 1.097); - assert_eq!(row.price, 332.64001); - assert_eq!(row.last_annual_dividend, 1.05); + assert_eq!(row.beta, Some(1.097)); + assert_eq!(row.price, Some(332.64001)); + assert_eq!(row.last_annual_dividend, Some(1.05)); assert_eq!(row.volume, 29_909_012.0); assert_eq!(row.exchange, "NASDAQ Global Select"); assert_eq!(row.exchange_short_name.as_str(), "NASDAQ"); - assert_eq!(row.country.as_str(), "US"); + assert_eq!(row.country.as_ref().unwrap().as_str(), "US"); assert!(!row.is_etf); - assert!(!row.is_fund); + assert_eq!(row.is_fund, Some(false)); assert!(row.is_actively_trading); let encoded = serde_json::to_value(row).unwrap(); @@ -51,7 +51,42 @@ fn screener_arrays_preserve_empty_multiple_unknown_and_large_integer_values() { assert_eq!(multiple[0].sector.as_str(), "Future Sector"); assert_eq!(multiple[1].market_cap, 4_294_967_296.0); assert_eq!(multiple[1].volume, 4_294_967_296.0); - assert!(multiple[1].is_fund); + assert_eq!(multiple[1].is_fund, Some(true)); assert_eq!(unknown.len(), 1); assert_eq!(unknown[0].symbol.as_str(), "AAPL"); } + +#[test] +fn screener_page_with_a_null_beta_row_decodes_every_row() { + let rows: Vec = serde_json::from_str(include_str!( + "fixtures/company_screener_null_beta_synthetic.json" + )) + .unwrap(); + + assert_eq!(rows.len(), 40); + assert_eq!(rows[37].symbol.as_str(), "R37"); + assert_eq!(rows[37].beta, None); + assert_eq!(rows[36].beta, Some(1.097)); +} + +#[test] +fn nullable_screener_members_decode_null_to_none_and_encode_null() { + let mut wire: serde_json::Value = + serde_json::from_str(include_str!("fixtures/company_screener.json")).unwrap(); + for member in ["beta", "price", "lastAnnualDividend", "country", "isFund"] { + wire[0][member] = serde_json::Value::Null; + } + let rows: Vec = serde_json::from_value(wire).unwrap(); + let row = &rows[0]; + + assert_eq!(row.beta, None); + assert_eq!(row.price, None); + assert_eq!(row.last_annual_dividend, None); + assert_eq!(row.country, None); + assert_eq!(row.is_fund, None); + + let encoded = serde_json::to_value(row).unwrap(); + for member in ["beta", "price", "lastAnnualDividend", "country", "isFund"] { + assert!(encoded[member].is_null(), "{member} must re-encode as null"); + } +} diff --git a/crates/libfmp/tests/search_responses.rs b/crates/libfmp/tests/search_responses.rs index 455cce94..9783f4d2 100644 --- a/crates/libfmp/tests/search_responses.rs +++ b/crates/libfmp/tests/search_responses.rs @@ -76,34 +76,34 @@ fn documented_exchange_variant_decodes_every_field_and_inverted_exchange_names() assert_eq!(rows.len(), 1); let row = &rows[0]; assert_eq!(row.symbol.as_str(), "AAPL"); - assert_eq!(row.price, 331.85501); + assert_eq!(row.price, Some(331.85501)); assert_eq!(row.beta, 1.097); assert_eq!(row.vol_avg, 55_309_000.0); assert_eq!(row.market_cap, 4_874_072_686_740.0); assert!(row.market_cap > f64::from(u32::MAX)); assert_eq!(row.last_div, 1.05); - assert_eq!(row.range, "201.5-344.57"); - assert_eq!(row.changes, -6.33498); + assert_eq!(row.range.as_deref(), Some("201.5-344.57")); + assert_eq!(row.changes, Some(-6.33498)); assert_eq!(row.company_name, "Apple Inc."); assert_eq!(row.currency.as_str(), "USD"); - assert_eq!(row.cik.as_str(), "0000320193"); - assert_eq!(row.isin.as_str(), "US0378331005"); - assert_eq!(row.cusip.as_str(), "037833100"); + assert_eq!(row.cik.as_ref().unwrap().as_str(), "0000320193"); + assert_eq!(row.isin.as_ref().unwrap().as_str(), "US0378331005"); + assert_eq!(row.cusip.as_ref().unwrap().as_str(), "037833100"); assert_eq!(row.exchange, "NASDAQ Global Select"); assert_eq!(row.exchange_short_name.as_str(), "NASDAQ"); assert_eq!(row.industry, "Consumer Electronics"); - assert_eq!(row.website, "https://www.apple.com"); + assert_eq!(row.website.as_deref(), Some("https://www.apple.com")); assert!(row.description.starts_with("Apple Inc. is a global")); - assert_eq!(row.ceo, "Timothy D. Cook"); + assert_eq!(row.ceo.as_deref(), Some("Timothy D. Cook")); assert_eq!(row.sector, "Technology"); assert_eq!(row.country.as_str(), "US"); - assert_eq!(row.full_time_employees, "166000"); - assert_eq!(row.phone, "(408) 996-1010"); - assert_eq!(row.address, "One Apple Park Way"); - assert_eq!(row.city, "Cupertino"); - assert_eq!(row.state, "CA"); - assert_eq!(row.zip, "95014"); - assert_eq!(row.dcf_diff, 191.60731); + assert_eq!(row.full_time_employees.as_deref(), Some("166000")); + assert_eq!(row.phone.as_deref(), Some("(408) 996-1010")); + assert_eq!(row.address.as_deref(), Some("One Apple Park Way")); + assert_eq!(row.city.as_deref(), Some("Cupertino")); + assert_eq!(row.state.as_deref(), Some("CA")); + assert_eq!(row.zip.as_deref(), Some("95014")); + assert_eq!(row.dcf_diff, Some(191.60731)); assert_eq!(row.dcf, 140.70269296445176); assert_eq!( row.image, @@ -145,3 +145,68 @@ fn search_arrays_preserve_empty_multiple_and_unknown_field_shapes() { assert_eq!(unknown.len(), 1); assert_eq!(unknown[0].symbol.as_str(), "AAPL"); } + +#[test] +fn nullable_exchange_variant_members_decode_null_and_empty_cusip_to_none() { + let nullable = [ + "price", + "range", + "changes", + "cik", + "isin", + "website", + "ceo", + "fullTimeEmployees", + "phone", + "address", + "city", + "state", + "zip", + "dcfDiff", + ]; + for cusip in [serde_json::Value::Null, serde_json::Value::from("")] { + let mut wire: serde_json::Value = + serde_json::from_str(include_str!("fixtures/search_exchange_variants.json")).unwrap(); + for member in nullable { + wire[0][member] = serde_json::Value::Null; + } + wire[0]["cusip"] = cusip; + let rows: Vec = serde_json::from_value(wire).unwrap(); + let row = &rows[0]; + + assert_eq!(row.price, None); + assert_eq!(row.range, None); + assert_eq!(row.changes, None); + assert_eq!(row.cik, None); + assert_eq!(row.isin, None); + assert_eq!(row.cusip, None); + assert_eq!(row.website, None); + assert_eq!(row.ceo, None); + assert_eq!(row.full_time_employees, None); + assert_eq!(row.phone, None); + assert_eq!(row.address, None); + assert_eq!(row.city, None); + assert_eq!(row.state, None); + assert_eq!(row.zip, None); + assert_eq!(row.dcf_diff, None); + + let encoded = serde_json::to_value(row).unwrap(); + for member in nullable.iter().chain(&["cusip"]) { + assert!(encoded[member].is_null(), "{member} must re-encode as null"); + } + } +} + +#[test] +fn empty_plain_string_members_stay_empty_strings() { + let mut wire: serde_json::Value = + serde_json::from_str(include_str!("fixtures/search_exchange_variants.json")).unwrap(); + for member in ["ceo", "state", "zip"] { + wire[0][member] = serde_json::Value::from(""); + } + let rows: Vec = serde_json::from_value(wire).unwrap(); + + assert_eq!(rows[0].ceo.as_deref(), Some("")); + assert_eq!(rows[0].state.as_deref(), Some("")); + assert_eq!(rows[0].zip.as_deref(), Some("")); +} diff --git a/sdk/go/decode_path_test.go b/sdk/go/decode_path_test.go index ae46a670..a325eeee 100644 --- a/sdk/go/decode_path_test.go +++ b/sdk/go/decode_path_test.go @@ -30,12 +30,14 @@ func decodeScreenerBody(t *testing.T, body []byte) *Error { func TestDecodeErrorNamesTheNullMemberRowAndMember(t *testing.T) { t.Parallel() - typed := decodeScreenerBody(t, readFixture(t, "company_screener_null_beta_synthetic.json")) + body := strings.Replace(string(readFixture(t, "company_screener_null_beta_synthetic.json")), + `"companyName": "Synthetic Row 37"`, `"companyName": null`, 1) + typed := decodeScreenerBody(t, []byte(body)) - if typed.Path != "/37/beta" || typed.DecodeKind != DecodeKindNull { - t.Fatalf("Path = %q, DecodeKind = %v, want /37/beta null", typed.Path, typed.DecodeKind) + if typed.Path != "/37/companyName" || typed.DecodeKind != DecodeKindNull { + t.Fatalf("Path = %q, DecodeKind = %v, want /37/companyName null", typed.Path, typed.DecodeKind) } - want := "successful response could not be decoded: null value at /37/beta (endpoint: company-screener): " + want := "successful response could not be decoded: null value at /37/companyName (endpoint: company-screener): " if !strings.HasPrefix(typed.Error(), want) { t.Fatalf("Error() = %.120q, want prefix %q", typed.Error(), want) } @@ -44,11 +46,11 @@ func TestDecodeErrorNamesTheNullMemberRowAndMember(t *testing.T) { func TestDecodeErrorNeverCarriesAStringMemberValue(t *testing.T) { t.Parallel() body := strings.Replace(string(readFixture(t, "company_screener_null_beta_synthetic.json")), - `"beta": null`, `"beta": "`+decodePathSentinel+`"`, 1) + "\"symbol\": \"R37\",\n \"volume\": 29909012", "\"symbol\": \"R37\",\n \"volume\": \""+decodePathSentinel+"\"", 1) typed := decodeScreenerBody(t, []byte(body)) - if typed.Path != "/37/beta" || typed.DecodeKind != DecodeKindWrongType { - t.Fatalf("Path = %q, DecodeKind = %v, want /37/beta wrong_type", typed.Path, typed.DecodeKind) + if typed.Path != "/37/volume" || typed.DecodeKind != DecodeKindWrongType { + t.Fatalf("Path = %q, DecodeKind = %v, want /37/volume wrong_type", typed.Path, typed.DecodeKind) } if typed.Body == nil || !typed.Body.Truncated { t.Fatalf("Body = %+v, want a truncated excerpt that ends before row 37", typed.Body) diff --git a/sdk/go/screener_models.go b/sdk/go/screener_models.go index bc365db2..247707af 100644 --- a/sdk/go/screener_models.go +++ b/sdk/go/screener_models.go @@ -13,42 +13,42 @@ import ( // CompanyScreenerResult is one worldwide company returned by the stock // screener. type CompanyScreenerResult struct { - Symbol string `json:"symbol"` - CompanyName string `json:"companyName"` - MarketCap float64 `json:"marketCap"` - Sector string `json:"sector"` - Industry string `json:"industry"` - Beta float64 `json:"beta"` - Price float64 `json:"price"` - LastAnnualDividend float64 `json:"lastAnnualDividend"` - Volume float64 `json:"volume"` - Exchange string `json:"exchange"` - ExchangeShortName string `json:"exchangeShortName"` - Country string `json:"country"` - IsETF bool `json:"isEtf"` - IsFund bool `json:"isFund"` - IsActivelyTrading bool `json:"isActivelyTrading"` + Symbol string `json:"symbol"` + CompanyName string `json:"companyName"` + MarketCap float64 `json:"marketCap"` + Sector string `json:"sector"` + Industry string `json:"industry"` + Beta *float64 `json:"beta"` + Price *float64 `json:"price"` + LastAnnualDividend *float64 `json:"lastAnnualDividend"` + Volume float64 `json:"volume"` + Exchange string `json:"exchange"` + ExchangeShortName string `json:"exchangeShortName"` + Country *string `json:"country"` + IsETF bool `json:"isEtf"` + IsFund *bool `json:"isFund"` + IsActivelyTrading bool `json:"isActivelyTrading"` } // companyScreenerResultShadow mirrors CompanyScreenerResult with a pointer or // raw value for every required member so a missing or null member is // observable after decoding. type companyScreenerResultShadow struct { - Symbol *string `json:"symbol"` - CompanyName *string `json:"companyName"` - MarketCap *float64 `json:"marketCap"` - Sector *string `json:"sector"` - Industry *string `json:"industry"` - Beta *float64 `json:"beta"` - Price *float64 `json:"price"` - LastAnnualDividend *float64 `json:"lastAnnualDividend"` - Volume *float64 `json:"volume"` - Exchange *string `json:"exchange"` - ExchangeShortName *string `json:"exchangeShortName"` - Country *string `json:"country"` - IsETF *bool `json:"isEtf"` - IsFund *bool `json:"isFund"` - IsActivelyTrading *bool `json:"isActivelyTrading"` + Symbol *string `json:"symbol"` + CompanyName *string `json:"companyName"` + MarketCap *float64 `json:"marketCap"` + Sector *string `json:"sector"` + Industry *string `json:"industry"` + Beta jsontext.Value `json:"beta"` + Price jsontext.Value `json:"price"` + LastAnnualDividend jsontext.Value `json:"lastAnnualDividend"` + Volume *float64 `json:"volume"` + Exchange *string `json:"exchange"` + ExchangeShortName *string `json:"exchangeShortName"` + Country jsontext.Value `json:"country"` + IsETF *bool `json:"isEtf"` + IsFund jsontext.Value `json:"isFund"` + IsActivelyTrading *bool `json:"isActivelyTrading"` } // UnmarshalJSONFrom decodes one JSON object and rejects it with a Decode @@ -70,11 +70,11 @@ func (m *CompanyScreenerResult) UnmarshalJSONFrom(dec *jsontext.Decoder) error { return missingMemberError("CompanyScreenerResult", "sector") case shadow.Industry == nil: return missingMemberError("CompanyScreenerResult", "industry") - case shadow.Beta == nil: + case len(shadow.Beta) == 0: return missingMemberError("CompanyScreenerResult", "beta") - case shadow.Price == nil: + case len(shadow.Price) == 0: return missingMemberError("CompanyScreenerResult", "price") - case shadow.LastAnnualDividend == nil: + case len(shadow.LastAnnualDividend) == 0: return missingMemberError("CompanyScreenerResult", "lastAnnualDividend") case shadow.Volume == nil: return missingMemberError("CompanyScreenerResult", "volume") @@ -82,30 +82,70 @@ func (m *CompanyScreenerResult) UnmarshalJSONFrom(dec *jsontext.Decoder) error { return missingMemberError("CompanyScreenerResult", "exchange") case shadow.ExchangeShortName == nil: return missingMemberError("CompanyScreenerResult", "exchangeShortName") - case shadow.Country == nil: + case len(shadow.Country) == 0: return missingMemberError("CompanyScreenerResult", "country") case shadow.IsETF == nil: return missingMemberError("CompanyScreenerResult", "isEtf") - case shadow.IsFund == nil: + case len(shadow.IsFund) == 0: return missingMemberError("CompanyScreenerResult", "isFund") case shadow.IsActivelyTrading == nil: return missingMemberError("CompanyScreenerResult", "isActivelyTrading") } + var beta *float64 + if shadow.Beta.Kind() != 'n' { + var value float64 + if err := json.Unmarshal(shadow.Beta, &value); err != nil { + return err + } + beta = &value + } + var price *float64 + if shadow.Price.Kind() != 'n' { + var value float64 + if err := json.Unmarshal(shadow.Price, &value); err != nil { + return err + } + price = &value + } + var lastAnnualDividend *float64 + if shadow.LastAnnualDividend.Kind() != 'n' { + var value float64 + if err := json.Unmarshal(shadow.LastAnnualDividend, &value); err != nil { + return err + } + lastAnnualDividend = &value + } + var country *string + if shadow.Country.Kind() != 'n' { + var value string + if err := json.Unmarshal(shadow.Country, &value); err != nil { + return err + } + country = &value + } + var isFund *bool + if shadow.IsFund.Kind() != 'n' { + var value bool + if err := json.Unmarshal(shadow.IsFund, &value); err != nil { + return err + } + isFund = &value + } *m = CompanyScreenerResult{ Symbol: *shadow.Symbol, CompanyName: *shadow.CompanyName, MarketCap: *shadow.MarketCap, Sector: *shadow.Sector, Industry: *shadow.Industry, - Beta: *shadow.Beta, - Price: *shadow.Price, - LastAnnualDividend: *shadow.LastAnnualDividend, + Beta: beta, + Price: price, + LastAnnualDividend: lastAnnualDividend, Volume: *shadow.Volume, Exchange: *shadow.Exchange, ExchangeShortName: *shadow.ExchangeShortName, - Country: *shadow.Country, + Country: country, IsETF: *shadow.IsETF, - IsFund: *shadow.IsFund, + IsFund: isFund, IsActivelyTrading: *shadow.IsActivelyTrading, } return nil diff --git a/sdk/go/screener_parity_test.go b/sdk/go/screener_parity_test.go index 633fbbdd..6b4aee31 100644 --- a/sdk/go/screener_parity_test.go +++ b/sdk/go/screener_parity_test.go @@ -4,6 +4,7 @@ import ( "encoding/json/v2" "errors" "math" + "reflect" "strings" "testing" ) @@ -31,18 +32,18 @@ func TestDocumentedCompanyScreenerResultDecodesExactValues(t *testing.T) { MarketCap: 4_885_602_246_714, Sector: "Technology", Industry: "Consumer Electronics", - Beta: 1.097, - Price: 332.64001, - LastAnnualDividend: 1.05, + Beta: new(1.097), + Price: new(332.64001), + LastAnnualDividend: new(1.05), Volume: 29_909_012, Exchange: "NASDAQ Global Select", ExchangeShortName: "NASDAQ", - Country: "US", + Country: new("US"), IsETF: false, - IsFund: false, + IsFund: new(false), IsActivelyTrading: true, } - if len(rows) != 1 || rows[0] != want { + if len(rows) != 1 || !reflect.DeepEqual(rows[0], want) { t.Fatalf("company_screener = %+v, want %+v", rows, want) } @@ -80,20 +81,20 @@ func TestScreenerArraysPreserveEmptyMultipleUnknownAndLargeIntegers(t *testing.T } first := CompanyScreenerResult{ Symbol: "BIG", CompanyName: "Beyond Float Precision Corp.", MarketCap: 9_007_199_254_740_992, - Sector: "Future Sector", Industry: "Future Industry", Beta: 0, Price: 0, LastAnnualDividend: 0, - Volume: math.MaxUint64, Exchange: "Future Exchange", ExchangeShortName: "NEXT", Country: "ZZ", - IsETF: false, IsFund: false, IsActivelyTrading: false, + Sector: "Future Sector", Industry: "Future Industry", Beta: new(0.0), Price: new(0.0), + LastAnnualDividend: new(0.0), Volume: math.MaxUint64, Exchange: "Future Exchange", ExchangeShortName: "NEXT", + Country: new("ZZ"), IsETF: false, IsFund: new(false), IsActivelyTrading: false, } - if multiple[0] != first || multiple[0].MarketCap < 1<<53 { + if !reflect.DeepEqual(multiple[0], first) || multiple[0].MarketCap < 1<<53 { t.Fatalf("company_screener_multiple[0] = %+v, want %+v", multiple[0], first) } second := CompanyScreenerResult{ Symbol: "FUND", CompanyName: "Example Fund", MarketCap: 4_294_967_296, - Sector: "Financial Services", Industry: "Asset Management", Beta: 1.25, Price: 42.5, LastAnnualDividend: 2.5, - Volume: 4_294_967_296, Exchange: "New York Stock Exchange", ExchangeShortName: "NYSE", Country: "US", - IsETF: false, IsFund: true, IsActivelyTrading: true, + Sector: "Financial Services", Industry: "Asset Management", Beta: new(1.25), Price: new(42.5), + LastAnnualDividend: new(2.5), Volume: 4_294_967_296, Exchange: "New York Stock Exchange", ExchangeShortName: "NYSE", + Country: new("US"), IsETF: false, IsFund: new(true), IsActivelyTrading: true, } - if multiple[1] != second { + if !reflect.DeepEqual(multiple[1], second) { t.Fatalf("company_screener_multiple[1] = %+v, want %+v", multiple[1], second) } @@ -109,6 +110,35 @@ func TestScreenerArraysPreserveEmptyMultipleUnknownAndLargeIntegers(t *testing.T } } +// Issue #368: FMP sends null for beta, price, lastAnnualDividend, country, and +// isFund on some rows; a page carrying one null beta row decodes whole, and +// every nullable member decodes null to nil and re-encodes it as null. +func TestScreenerNullableMembersDecodeNullToNil(t *testing.T) { + t.Parallel() + page := assertFixtureParity[CompanyScreenerResult](t, "company_screener_null_beta_synthetic.json") + if len(page) != 40 || page[37].Beta != nil || page[36].Beta == nil || *page[36].Beta != 1.097 { + t.Fatalf("null beta page: %d rows, row 37 beta = %v, row 36 beta = %v", len(page), page[37].Beta, page[36].Beta) + } + + row := string(readFixture(t, "company_screener.json")) + for _, member := range []string{`"beta": 1.097`, `"price": 332.64001`, `"lastAnnualDividend": 1.05`, `"country": "US"`, `"isFund": false`} { + name, _, _ := strings.Cut(member, ":") + row = strings.Replace(row, member, name+": null", 1) + } + var rows []CompanyScreenerResult + if err := json.Unmarshal([]byte(row), &rows); err != nil { + t.Fatal(err) + } + got := rows[0] + if got.Beta != nil || got.Price != nil || got.LastAnnualDividend != nil || got.Country != nil || got.IsFund != nil { + t.Fatalf("null members = %+v, want nil pointers", got) + } + encoded, err := json.Marshal(got) + if err != nil || !strings.Contains(string(encoded), `"beta":null`) || !strings.Contains(string(encoded), `"isFund":null`) { + t.Fatalf("re-encoded = %s, %v", encoded, err) + } +} + // The missing-required-member path of the generated decoder, once for this // domain: serde rejects a missing and a null member alike, and the error // names the model and the first offending member. diff --git a/sdk/go/screener_test.go b/sdk/go/screener_test.go index f4861922..198f37dc 100644 --- a/sdk/go/screener_test.go +++ b/sdk/go/screener_test.go @@ -83,7 +83,7 @@ func TestScreenerCompaniesUsesExactPathAndWireParameterOrder(t *testing.T) { } // The direct-transport contract of screener_endpoint.rs. direct, err := client.Screener.Companies(ctx, NewCompanyScreenerQuery().WithSector("Technology").WithLimit(1_000)) - if err != nil || len(direct) != 2 || direct[0].Volume != math.MaxUint64 || !direct[1].IsFund { + if err != nil || len(direct) != 2 || direct[0].Volume != math.MaxUint64 || direct[1].IsFund == nil || !*direct[1].IsFund { t.Fatalf("Companies direct = %+v, %v", direct, err) } diff --git a/sdk/go/search_models.go b/sdk/go/search_models.go index 68c81874..ebb7db20 100644 --- a/sdk/go/search_models.go +++ b/sdk/go/search_models.go @@ -258,69 +258,27 @@ func (m *ISINSearchResult) UnmarshalJSONFrom(dec *jsontext.Decoder) error { // ExchangeVariant is one exchange listing returned by exchange-variants // search. type ExchangeVariant struct { - Symbol string `json:"symbol"` - Price float64 `json:"price"` - Beta float64 `json:"beta"` - VolAvg float64 `json:"volAvg"` - MarketCap float64 `json:"mktCap"` - LastDiv float64 `json:"lastDiv"` - Range string `json:"range"` - Changes float64 `json:"changes"` - CompanyName string `json:"companyName"` - Currency string `json:"currency"` - CIK string `json:"cik"` - ISIN string `json:"isin"` - CUSIP string `json:"cusip"` - Exchange string `json:"exchange"` - ExchangeShortName string `json:"exchangeShortName"` - Industry string `json:"industry"` - Website string `json:"website"` - Description string `json:"description"` - Ceo string `json:"ceo"` - Sector string `json:"sector"` - Country string `json:"country"` - FullTimeEmployees string `json:"fullTimeEmployees"` - Phone string `json:"phone"` - Address string `json:"address"` - City string `json:"city"` - State string `json:"state"` - Zip string `json:"zip"` - DCFDiff float64 `json:"dcfDiff"` - DCF float64 `json:"dcf"` - Image string `json:"image"` - IPODate Date `json:"ipoDate"` - DefaultImage bool `json:"defaultImage"` - IsETF bool `json:"isEtf"` - IsActivelyTrading bool `json:"isActivelyTrading"` - IsAdr bool `json:"isAdr"` - IsFund bool `json:"isFund"` -} - -// exchangeVariantShadow mirrors ExchangeVariant with a pointer or raw value -// for every required member so a missing or null member is observable after -// decoding. -type exchangeVariantShadow struct { - Symbol *string `json:"symbol"` + Symbol string `json:"symbol"` Price *float64 `json:"price"` - Beta *float64 `json:"beta"` - VolAvg *float64 `json:"volAvg"` - MarketCap *float64 `json:"mktCap"` - LastDiv *float64 `json:"lastDiv"` + Beta float64 `json:"beta"` + VolAvg float64 `json:"volAvg"` + MarketCap float64 `json:"mktCap"` + LastDiv float64 `json:"lastDiv"` Range *string `json:"range"` Changes *float64 `json:"changes"` - CompanyName *string `json:"companyName"` - Currency *string `json:"currency"` + CompanyName string `json:"companyName"` + Currency string `json:"currency"` CIK *string `json:"cik"` ISIN *string `json:"isin"` CUSIP *string `json:"cusip"` - Exchange *string `json:"exchange"` - ExchangeShortName *string `json:"exchangeShortName"` - Industry *string `json:"industry"` + Exchange string `json:"exchange"` + ExchangeShortName string `json:"exchangeShortName"` + Industry string `json:"industry"` Website *string `json:"website"` - Description *string `json:"description"` + Description string `json:"description"` Ceo *string `json:"ceo"` - Sector *string `json:"sector"` - Country *string `json:"country"` + Sector string `json:"sector"` + Country string `json:"country"` FullTimeEmployees *string `json:"fullTimeEmployees"` Phone *string `json:"phone"` Address *string `json:"address"` @@ -328,14 +286,56 @@ type exchangeVariantShadow struct { State *string `json:"state"` Zip *string `json:"zip"` DCFDiff *float64 `json:"dcfDiff"` - DCF *float64 `json:"dcf"` - Image *string `json:"image"` - IPODate *Date `json:"ipoDate"` - DefaultImage *bool `json:"defaultImage"` - IsETF *bool `json:"isEtf"` - IsActivelyTrading *bool `json:"isActivelyTrading"` - IsAdr *bool `json:"isAdr"` - IsFund *bool `json:"isFund"` + DCF float64 `json:"dcf"` + Image string `json:"image"` + IPODate Date `json:"ipoDate"` + DefaultImage bool `json:"defaultImage"` + IsETF bool `json:"isEtf"` + IsActivelyTrading bool `json:"isActivelyTrading"` + IsAdr bool `json:"isAdr"` + IsFund bool `json:"isFund"` +} + +// exchangeVariantShadow mirrors ExchangeVariant with a pointer or raw value +// for every required member so a missing or null member is observable after +// decoding. +type exchangeVariantShadow struct { + Symbol *string `json:"symbol"` + Price jsontext.Value `json:"price"` + Beta *float64 `json:"beta"` + VolAvg *float64 `json:"volAvg"` + MarketCap *float64 `json:"mktCap"` + LastDiv *float64 `json:"lastDiv"` + Range jsontext.Value `json:"range"` + Changes jsontext.Value `json:"changes"` + CompanyName *string `json:"companyName"` + Currency *string `json:"currency"` + CIK jsontext.Value `json:"cik"` + ISIN jsontext.Value `json:"isin"` + CUSIP jsontext.Value `json:"cusip"` + Exchange *string `json:"exchange"` + ExchangeShortName *string `json:"exchangeShortName"` + Industry *string `json:"industry"` + Website jsontext.Value `json:"website"` + Description *string `json:"description"` + Ceo jsontext.Value `json:"ceo"` + Sector *string `json:"sector"` + Country *string `json:"country"` + FullTimeEmployees jsontext.Value `json:"fullTimeEmployees"` + Phone jsontext.Value `json:"phone"` + Address jsontext.Value `json:"address"` + City jsontext.Value `json:"city"` + State jsontext.Value `json:"state"` + Zip jsontext.Value `json:"zip"` + DCFDiff jsontext.Value `json:"dcfDiff"` + DCF *float64 `json:"dcf"` + Image *string `json:"image"` + IPODate *Date `json:"ipoDate"` + DefaultImage *bool `json:"defaultImage"` + IsETF *bool `json:"isEtf"` + IsActivelyTrading *bool `json:"isActivelyTrading"` + IsAdr *bool `json:"isAdr"` + IsFund *bool `json:"isFund"` } // UnmarshalJSONFrom decodes one JSON object and rejects it with a Decode @@ -349,7 +349,7 @@ func (m *ExchangeVariant) UnmarshalJSONFrom(dec *jsontext.Decoder) error { switch { case shadow.Symbol == nil: return missingMemberError("ExchangeVariant", "symbol") - case shadow.Price == nil: + case len(shadow.Price) == 0: return missingMemberError("ExchangeVariant", "price") case shadow.Beta == nil: return missingMemberError("ExchangeVariant", "beta") @@ -359,19 +359,19 @@ func (m *ExchangeVariant) UnmarshalJSONFrom(dec *jsontext.Decoder) error { return missingMemberError("ExchangeVariant", "mktCap") case shadow.LastDiv == nil: return missingMemberError("ExchangeVariant", "lastDiv") - case shadow.Range == nil: + case len(shadow.Range) == 0: return missingMemberError("ExchangeVariant", "range") - case shadow.Changes == nil: + case len(shadow.Changes) == 0: return missingMemberError("ExchangeVariant", "changes") case shadow.CompanyName == nil: return missingMemberError("ExchangeVariant", "companyName") case shadow.Currency == nil: return missingMemberError("ExchangeVariant", "currency") - case shadow.CIK == nil: + case len(shadow.CIK) == 0: return missingMemberError("ExchangeVariant", "cik") - case shadow.ISIN == nil: + case len(shadow.ISIN) == 0: return missingMemberError("ExchangeVariant", "isin") - case shadow.CUSIP == nil: + case len(shadow.CUSIP) == 0: return missingMemberError("ExchangeVariant", "cusip") case shadow.Exchange == nil: return missingMemberError("ExchangeVariant", "exchange") @@ -379,29 +379,29 @@ func (m *ExchangeVariant) UnmarshalJSONFrom(dec *jsontext.Decoder) error { return missingMemberError("ExchangeVariant", "exchangeShortName") case shadow.Industry == nil: return missingMemberError("ExchangeVariant", "industry") - case shadow.Website == nil: + case len(shadow.Website) == 0: return missingMemberError("ExchangeVariant", "website") case shadow.Description == nil: return missingMemberError("ExchangeVariant", "description") - case shadow.Ceo == nil: + case len(shadow.Ceo) == 0: return missingMemberError("ExchangeVariant", "ceo") case shadow.Sector == nil: return missingMemberError("ExchangeVariant", "sector") case shadow.Country == nil: return missingMemberError("ExchangeVariant", "country") - case shadow.FullTimeEmployees == nil: + case len(shadow.FullTimeEmployees) == 0: return missingMemberError("ExchangeVariant", "fullTimeEmployees") - case shadow.Phone == nil: + case len(shadow.Phone) == 0: return missingMemberError("ExchangeVariant", "phone") - case shadow.Address == nil: + case len(shadow.Address) == 0: return missingMemberError("ExchangeVariant", "address") - case shadow.City == nil: + case len(shadow.City) == 0: return missingMemberError("ExchangeVariant", "city") - case shadow.State == nil: + case len(shadow.State) == 0: return missingMemberError("ExchangeVariant", "state") - case shadow.Zip == nil: + case len(shadow.Zip) == 0: return missingMemberError("ExchangeVariant", "zip") - case shadow.DCFDiff == nil: + case len(shadow.DCFDiff) == 0: return missingMemberError("ExchangeVariant", "dcfDiff") case shadow.DCF == nil: return missingMemberError("ExchangeVariant", "dcf") @@ -420,35 +420,157 @@ func (m *ExchangeVariant) UnmarshalJSONFrom(dec *jsontext.Decoder) error { case shadow.IsFund == nil: return missingMemberError("ExchangeVariant", "isFund") } + var price *float64 + if shadow.Price.Kind() != 'n' { + var value float64 + if err := json.Unmarshal(shadow.Price, &value); err != nil { + return err + } + price = &value + } + var range_ *string + if shadow.Range.Kind() != 'n' { + var value string + if err := json.Unmarshal(shadow.Range, &value); err != nil { + return err + } + range_ = &value + } + var changes *float64 + if shadow.Changes.Kind() != 'n' { + var value float64 + if err := json.Unmarshal(shadow.Changes, &value); err != nil { + return err + } + changes = &value + } + var cik *string + if shadow.CIK.Kind() != 'n' { + var value string + if err := json.Unmarshal(shadow.CIK, &value); err != nil { + return err + } + cik = &value + } + var isin *string + if shadow.ISIN.Kind() != 'n' { + var value string + if err := json.Unmarshal(shadow.ISIN, &value); err != nil { + return err + } + isin = &value + } + var cusip *string + if shadow.CUSIP.Kind() != 'n' { + var value string + if err := json.Unmarshal(shadow.CUSIP, &value); err != nil { + return err + } + if value != "" { + cusip = &value + } + } + var website *string + if shadow.Website.Kind() != 'n' { + var value string + if err := json.Unmarshal(shadow.Website, &value); err != nil { + return err + } + website = &value + } + var ceo *string + if shadow.Ceo.Kind() != 'n' { + var value string + if err := json.Unmarshal(shadow.Ceo, &value); err != nil { + return err + } + ceo = &value + } + var fullTimeEmployees *string + if shadow.FullTimeEmployees.Kind() != 'n' { + var value string + if err := json.Unmarshal(shadow.FullTimeEmployees, &value); err != nil { + return err + } + fullTimeEmployees = &value + } + var phone *string + if shadow.Phone.Kind() != 'n' { + var value string + if err := json.Unmarshal(shadow.Phone, &value); err != nil { + return err + } + phone = &value + } + var address *string + if shadow.Address.Kind() != 'n' { + var value string + if err := json.Unmarshal(shadow.Address, &value); err != nil { + return err + } + address = &value + } + var city *string + if shadow.City.Kind() != 'n' { + var value string + if err := json.Unmarshal(shadow.City, &value); err != nil { + return err + } + city = &value + } + var state *string + if shadow.State.Kind() != 'n' { + var value string + if err := json.Unmarshal(shadow.State, &value); err != nil { + return err + } + state = &value + } + var zip *string + if shadow.Zip.Kind() != 'n' { + var value string + if err := json.Unmarshal(shadow.Zip, &value); err != nil { + return err + } + zip = &value + } + var dcfDiff *float64 + if shadow.DCFDiff.Kind() != 'n' { + var value float64 + if err := json.Unmarshal(shadow.DCFDiff, &value); err != nil { + return err + } + dcfDiff = &value + } *m = ExchangeVariant{ Symbol: *shadow.Symbol, - Price: *shadow.Price, + Price: price, Beta: *shadow.Beta, VolAvg: *shadow.VolAvg, MarketCap: *shadow.MarketCap, LastDiv: *shadow.LastDiv, - Range: *shadow.Range, - Changes: *shadow.Changes, + Range: range_, + Changes: changes, CompanyName: *shadow.CompanyName, Currency: *shadow.Currency, - CIK: *shadow.CIK, - ISIN: *shadow.ISIN, - CUSIP: *shadow.CUSIP, + CIK: cik, + ISIN: isin, + CUSIP: cusip, Exchange: *shadow.Exchange, ExchangeShortName: *shadow.ExchangeShortName, Industry: *shadow.Industry, - Website: *shadow.Website, + Website: website, Description: *shadow.Description, - Ceo: *shadow.Ceo, + Ceo: ceo, Sector: *shadow.Sector, Country: *shadow.Country, - FullTimeEmployees: *shadow.FullTimeEmployees, - Phone: *shadow.Phone, - Address: *shadow.Address, - City: *shadow.City, - State: *shadow.State, - Zip: *shadow.Zip, - DCFDiff: *shadow.DCFDiff, + FullTimeEmployees: fullTimeEmployees, + Phone: phone, + Address: address, + City: city, + State: state, + Zip: zip, + DCFDiff: dcfDiff, DCF: *shadow.DCF, Image: *shadow.Image, IPODate: *shadow.IPODate, diff --git a/sdk/go/search_parity_test.go b/sdk/go/search_parity_test.go index dedebbd1..ed1ad697 100644 --- a/sdk/go/search_parity_test.go +++ b/sdk/go/search_parity_test.go @@ -4,6 +4,7 @@ import ( "encoding/json/v2" "errors" "math" + "reflect" "strings" "testing" ) @@ -111,17 +112,17 @@ func TestDocumentedExchangeVariantDecodesExactValues(t *testing.T) { } row := rows[0] want := ExchangeVariant{ - Symbol: "AAPL", Price: 331.85501, Beta: 1.097, VolAvg: 55_309_000, MarketCap: 4_874_072_686_740, - LastDiv: 1.05, Range: "201.5-344.57", Changes: -6.33498, CompanyName: "Apple Inc.", Currency: "USD", - CIK: "0000320193", ISIN: "US0378331005", CUSIP: "037833100", Exchange: "NASDAQ Global Select", - ExchangeShortName: "NASDAQ", Industry: "Consumer Electronics", Website: "https://www.apple.com", - Description: row.Description, Ceo: "Timothy D. Cook", Sector: "Technology", Country: "US", - FullTimeEmployees: "166000", Phone: "(408) 996-1010", Address: "One Apple Park Way", City: "Cupertino", - State: "CA", Zip: "95014", DCFDiff: 191.60731, DCF: 140.70269296445176, + Symbol: "AAPL", Price: new(331.85501), Beta: 1.097, VolAvg: 55_309_000, MarketCap: 4_874_072_686_740, + LastDiv: 1.05, Range: new("201.5-344.57"), Changes: new(-6.33498), CompanyName: "Apple Inc.", Currency: "USD", + CIK: new("0000320193"), ISIN: new("US0378331005"), CUSIP: new("037833100"), Exchange: "NASDAQ Global Select", + ExchangeShortName: "NASDAQ", Industry: "Consumer Electronics", Website: new("https://www.apple.com"), + Description: row.Description, Ceo: new("Timothy D. Cook"), Sector: "Technology", Country: "US", + FullTimeEmployees: new("166000"), Phone: new("(408) 996-1010"), Address: new("One Apple Park Way"), + City: new("Cupertino"), State: new("CA"), Zip: new("95014"), DCFDiff: new(191.60731), DCF: 140.70269296445176, Image: "https://images.financialmodelingprep.com/symbol/AAPL.png", IPODate: mustParseDate(t, "1980-12-12"), DefaultImage: false, IsETF: false, IsActivelyTrading: true, IsAdr: false, IsFund: false, } - if row != want { + if !reflect.DeepEqual(row, want) { t.Fatalf("search_exchange_variants[0] = %+v, want %+v", row, want) } if !strings.HasPrefix(row.Description, "Apple Inc. is a global") || row.MarketCap <= math.MaxUint32 { @@ -145,6 +146,44 @@ func TestDocumentedExchangeVariantDecodesExactValues(t *testing.T) { } } +// Issue #368: FMP sends null for 15 ExchangeVariant members on some listings +// and "" for cusip; both decode to nil, and nil re-encodes as null. +func TestExchangeVariantNullableMembersDecodeNullAndEmptyCUSIPToNil(t *testing.T) { + t.Parallel() + var wire []map[string]any + if err := json.Unmarshal(readFixture(t, "search_exchange_variants.json"), &wire); err != nil { + t.Fatal(err) + } + for _, member := range []string{"price", "range", "changes", "cik", "isin", "website", "ceo", "fullTimeEmployees", + "phone", "address", "city", "state", "zip", "dcfDiff"} { + wire[0][member] = nil + } + for _, cusip := range []any{nil, ""} { + wire[0]["cusip"] = cusip + body, err := json.Marshal(wire) + if err != nil { + t.Fatal(err) + } + var rows []ExchangeVariant + if err := json.Unmarshal(body, &rows); err != nil { + t.Fatalf("cusip %q: %v", cusip, err) + } + row := rows[0] + for name, member := range map[string]any{"price": row.Price, "range": row.Range, "changes": row.Changes, + "cik": row.CIK, "isin": row.ISIN, "cusip": row.CUSIP, "website": row.Website, "ceo": row.Ceo, + "fullTimeEmployees": row.FullTimeEmployees, "phone": row.Phone, "address": row.Address, "city": row.City, + "state": row.State, "zip": row.Zip, "dcfDiff": row.DCFDiff} { + if !reflect.ValueOf(member).IsNil() { + t.Fatalf("cusip %q: %s = %v, want nil", cusip, name, member) + } + } + encoded, err := json.Marshal(row) + if err != nil || !strings.Contains(string(encoded), `"cusip":null`) || !strings.Contains(string(encoded), `"dcfDiff":null`) { + t.Fatalf("re-encoded = %s, %v", encoded, err) + } + } +} + // The missing-required-member path of the generated decoders, once for this // domain: serde rejects a missing and a null member alike, and a malformed // date never decodes into a Date member. From ffc731d6b92908b1be0ad2072796eb5aa1e11dc7 Mon Sep 17 00:00:00 2001 From: benbenbang Date: Wed, 30 Sep 2026 20:57:07 +0800 Subject: [PATCH 2/2] docs(crates): update fixture README for issue #368 optional beta Document that issue #368 made the `beta` field optional, so the `company_screener_null_beta_synthetic.json` fixture now proves the null row decodes across Rust, Go, and Python. Note the decode-path proof also swaps a null `companyName` into row 37 and a string `volume` to confirm the offending value never reaches the error path. --- crates/libfmp/tests/fixtures/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/libfmp/tests/fixtures/README.md b/crates/libfmp/tests/fixtures/README.md index 3330ade8..2e5c4fd5 100644 --- a/crates/libfmp/tests/fixtures/README.md +++ b/crates/libfmp/tests/fixtures/README.md @@ -40,9 +40,11 @@ network connection. - **Decode-path proof.** `company_screener_null_beta_synthetic.json` is the one null variant kept as a file: 40 documented rows with `"beta": null` in row 37, so the offending member lands past the `MAX_SAFE_BODY_BYTES` body - excerpt and the Go and Python suites can share it. It is the issue #366 - decode-path proof in `decode_path.rs`, `sdk/go/decode_path_test.go`, and - the fmp-py screener tests, which also swap in a string value to prove it + excerpt and the Go and Python suites can share it. Since issue #368 made + `beta` optional, the file proves the null row decodes in all three + languages; the issue #366 decode-path proof in `decode_path.rs`, + `sdk/go/decode_path_test.go`, and the fmp-py screener tests now swaps a + null `companyName` into row 37, and a string `volume` to prove the value never reaches the error. - **Observed-shape reproductions.** `quote_short_fractional_volume.json` reproduces the one live `quote-short` row that motivated issue #337 (seen