diff --git a/crates/fmp-py/python/fmp/_native/congressional/__init__.pyi b/crates/fmp-py/python/fmp/_native/congressional/__init__.pyi index 2e56c9e..45e0d12 100644 --- a/crates/fmp-py/python/fmp/_native/congressional/__init__.pyi +++ b/crates/fmp-py/python/fmp/_native/congressional/__init__.pyi @@ -22,9 +22,11 @@ class CongressionalDebtDetails: __match_args__: tuple[typing.Literal["date_incurred"]] @property - def date_incurred(self) -> builtins.str: ... + def date_incurred(self) -> typing.Optional[builtins.str]: ... def __eq__(self, other: builtins.object, /) -> builtins.bool: ... - def __new__(cls, *, date_incurred: builtins.str) -> CongressionalDebtDetails: ... + def __new__( + cls, *, date_incurred: typing.Optional[builtins.str] + ) -> CongressionalDebtDetails: ... def __getnewargs_ex__(self) -> tuple[tuple, dict]: ... def __repr__(self) -> builtins.str: ... def to_dict(self) -> builtins.dict[builtins.str, typing.Any]: @@ -68,15 +70,15 @@ class CongressionalMemberNetWorth: @property def section(self) -> builtins.str: ... @property - def category(self) -> builtins.str: ... + def category(self) -> typing.Optional[builtins.str]: ... @property - def name(self) -> builtins.str: ... + def name(self) -> typing.Optional[builtins.str]: ... @property def asset_type(self) -> builtins.str: ... @property def income_type(self) -> typing.Optional[builtins.str]: ... @property - def owner(self) -> builtins.str: ... + def owner(self) -> typing.Optional[builtins.str]: ... @property def comment(self) -> typing.Optional[builtins.str]: ... @property @@ -84,7 +86,7 @@ class CongressionalMemberNetWorth: @property def value_range(self) -> typing.Optional[CongressionalNetWorthRange]: ... @property - def value(self) -> builtins.int: ... + def value(self) -> typing.Optional[builtins.float]: ... @property def income_range(self) -> typing.Optional[CongressionalNetWorthRange]: ... @property @@ -100,15 +102,15 @@ class CongressionalMemberNetWorth: year: builtins.int, filing_date: datetime.date, section: builtins.str, - category: builtins.str, - name: builtins.str, + category: typing.Optional[builtins.str], + name: typing.Optional[builtins.str], asset_type: builtins.str, income_type: typing.Optional[builtins.str], - owner: builtins.str, + owner: typing.Optional[builtins.str], comment: typing.Optional[builtins.str], debt_details: typing.Optional[CongressionalDebtDetails], value_range: typing.Optional[CongressionalNetWorthRange], - value: builtins.int, + value: typing.Optional[builtins.float], income_range: typing.Optional[CongressionalNetWorthRange], income: typing.Optional[builtins.str], link: builtins.str, @@ -147,47 +149,47 @@ class CongressionalMemberNetWorthAggregate: @property def year(self) -> builtins.int: ... @property - def total(self) -> builtins.int: ... + def total(self) -> builtins.float: ... @property - def real_estate_liabilities(self) -> builtins.int: ... + def real_estate_liabilities(self) -> typing.Optional[builtins.float]: ... @property - def cash_and_cash_equivalents(self) -> builtins.int: ... + def cash_and_cash_equivalents(self) -> builtins.float: ... @property - def business_and_self_employment(self) -> builtins.int: ... + def business_and_self_employment(self) -> typing.Optional[builtins.float]: ... @property - def real_estate(self) -> builtins.int: ... + def real_estate(self) -> typing.Optional[builtins.float]: ... @property - def ownership_interest(self) -> builtins.int: ... + def ownership_interest(self) -> typing.Optional[builtins.float]: ... @property - def stock(self) -> builtins.int: ... + def stock(self) -> typing.Optional[builtins.float]: ... @property - def options(self) -> builtins.int: ... + def options(self) -> typing.Optional[builtins.float]: ... @property - def revolving_and_credit_lines(self) -> builtins.int: ... + def revolving_and_credit_lines(self) -> typing.Optional[builtins.float]: ... @property - def asset_backed_securities(self) -> builtins.int: ... + def asset_backed_securities(self) -> typing.Optional[builtins.float]: ... @property - def business_liabilities(self) -> builtins.int: ... + def business_liabilities(self) -> typing.Optional[builtins.float]: ... @property - def mutual_funds_and_etfs(self) -> builtins.int: ... + def mutual_funds_and_etfs(self) -> builtins.float: ... def __eq__(self, other: builtins.object, /) -> builtins.bool: ... def __new__( cls, *, member_id: builtins.str, year: builtins.int, - total: builtins.int, - real_estate_liabilities: builtins.int, - cash_and_cash_equivalents: builtins.int, - business_and_self_employment: builtins.int, - real_estate: builtins.int, - ownership_interest: builtins.int, - stock: builtins.int, - options: builtins.int, - revolving_and_credit_lines: builtins.int, - asset_backed_securities: builtins.int, - business_liabilities: builtins.int, - mutual_funds_and_etfs: builtins.int, + total: builtins.float, + real_estate_liabilities: typing.Optional[builtins.float], + cash_and_cash_equivalents: builtins.float, + business_and_self_employment: typing.Optional[builtins.float], + real_estate: typing.Optional[builtins.float], + ownership_interest: typing.Optional[builtins.float], + stock: typing.Optional[builtins.float], + options: typing.Optional[builtins.float], + revolving_and_credit_lines: typing.Optional[builtins.float], + asset_backed_securities: typing.Optional[builtins.float], + business_liabilities: typing.Optional[builtins.float], + mutual_funds_and_etfs: builtins.float, ) -> CongressionalMemberNetWorthAggregate: ... def __getnewargs_ex__(self) -> tuple[tuple, dict]: ... def __repr__(self) -> builtins.str: ... @@ -451,10 +453,10 @@ class CongressionalNetWorthRange: @property def min(self) -> builtins.int: ... @property - def max(self) -> builtins.int: ... + def max(self) -> typing.Optional[builtins.int]: ... def __eq__(self, other: builtins.object, /) -> builtins.bool: ... def __new__( - cls, *, min: builtins.int, max: builtins.int + cls, *, min: builtins.int, max: typing.Optional[builtins.int] ) -> CongressionalNetWorthRange: ... def __getnewargs_ex__(self) -> tuple[tuple, dict]: ... def __repr__(self) -> builtins.str: ... diff --git a/crates/fmp-py/src/models/congressional.rs b/crates/fmp-py/src/models/congressional.rs index 96a3031..9e128a2 100644 --- a/crates/fmp-py/src/models/congressional.rs +++ b/crates/fmp-py/src/models/congressional.rs @@ -715,7 +715,7 @@ pub(crate) struct CongressionalNetWorthRange { #[pyo3(get)] pub min: i64, #[pyo3(get)] - pub max: i64, + pub max: Option, } #[gen_stub_pymethods] @@ -725,7 +725,7 @@ impl CongressionalNetWorthRange { #[allow(clippy::too_many_arguments)] #[allow(clippy::fn_params_excessive_bools)] #[pyo3(signature = (*, min, max))] - fn new(min: i64, max: i64) -> Self { + fn new(min: i64, max: Option) -> Self { Self { min, max } } @@ -798,7 +798,7 @@ impl crate::models::convert::DictValue for CongressionalNetWorthRange { #[derive(Clone, PartialEq)] pub(crate) struct CongressionalDebtDetails { #[pyo3(get)] - pub date_incurred: String, + pub date_incurred: Option, } #[gen_stub_pymethods] @@ -808,7 +808,7 @@ impl CongressionalDebtDetails { #[allow(clippy::too_many_arguments)] #[allow(clippy::fn_params_excessive_bools)] #[pyo3(signature = (*, date_incurred))] - fn new(date_incurred: String) -> Self { + fn new(date_incurred: Option) -> Self { Self { date_incurred } } @@ -860,7 +860,7 @@ impl CongressionalDebtDetails { impl From for CongressionalDebtDetails { fn from(value: libfmp::responses::congressional::CongressionalDebtDetails) -> Self { Self { - date_incurred: value.date_incurred.0, + date_incurred: value.date_incurred.map(|value| value.0), } } } @@ -886,15 +886,15 @@ pub(crate) struct CongressionalMemberNetWorth { #[pyo3(get)] pub section: String, #[pyo3(get)] - pub category: String, + pub category: Option, #[pyo3(get)] - pub name: String, + pub name: Option, #[pyo3(get)] pub asset_type: String, #[pyo3(get)] pub income_type: Option, #[pyo3(get)] - pub owner: String, + pub owner: Option, #[pyo3(get)] pub comment: Option, #[pyo3(get)] @@ -902,7 +902,7 @@ pub(crate) struct CongressionalMemberNetWorth { #[pyo3(get)] pub value_range: Option, #[pyo3(get)] - pub value: i64, + pub value: Option, #[pyo3(get)] pub income_range: Option, income: Option<::serde_json::Value>, @@ -923,15 +923,15 @@ impl CongressionalMemberNetWorth { year: u32, filing_date: ::chrono::NaiveDate, section: String, - category: String, - name: String, + category: Option, + name: Option, asset_type: String, income_type: Option, - owner: String, + owner: Option, comment: Option, debt_details: Option, value_range: Option, - value: i64, + value: Option, income_range: Option, income: Option, link: String, @@ -1219,29 +1219,29 @@ pub(crate) struct CongressionalMemberNetWorthAggregate { #[pyo3(get)] pub year: u32, #[pyo3(get)] - pub total: i64, + pub total: f64, #[pyo3(get)] - pub real_estate_liabilities: i64, + pub real_estate_liabilities: Option, #[pyo3(get)] - pub cash_and_cash_equivalents: i64, + pub cash_and_cash_equivalents: f64, #[pyo3(get)] - pub business_and_self_employment: i64, + pub business_and_self_employment: Option, #[pyo3(get)] - pub real_estate: i64, + pub real_estate: Option, #[pyo3(get)] - pub ownership_interest: i64, + pub ownership_interest: Option, #[pyo3(get)] - pub stock: i64, + pub stock: Option, #[pyo3(get)] - pub options: i64, + pub options: Option, #[pyo3(get)] - pub revolving_and_credit_lines: i64, + pub revolving_and_credit_lines: Option, #[pyo3(get)] - pub asset_backed_securities: i64, + pub asset_backed_securities: Option, #[pyo3(get)] - pub business_liabilities: i64, + pub business_liabilities: Option, #[pyo3(get)] - pub mutual_funds_and_etfs: i64, + pub mutual_funds_and_etfs: f64, } #[gen_stub_pymethods] @@ -1254,18 +1254,18 @@ impl CongressionalMemberNetWorthAggregate { fn new( member_id: String, year: u32, - total: i64, - real_estate_liabilities: i64, - cash_and_cash_equivalents: i64, - business_and_self_employment: i64, - real_estate: i64, - ownership_interest: i64, - stock: i64, - options: i64, - revolving_and_credit_lines: i64, - asset_backed_securities: i64, - business_liabilities: i64, - mutual_funds_and_etfs: i64, + total: f64, + real_estate_liabilities: Option, + cash_and_cash_equivalents: f64, + business_and_self_employment: Option, + real_estate: Option, + ownership_interest: Option, + stock: Option, + options: Option, + revolving_and_credit_lines: Option, + asset_backed_securities: Option, + business_liabilities: Option, + mutual_funds_and_etfs: f64, ) -> Self { Self { member_id, diff --git a/crates/fmp-py/tests/test_congressional_b.py b/crates/fmp-py/tests/test_congressional_b.py index e477f1d..1a832ae 100644 --- a/crates/fmp-py/tests/test_congressional_b.py +++ b/crates/fmp-py/tests/test_congressional_b.py @@ -285,3 +285,59 @@ def test_profiles_decode_null_image_to_none(client: Any, fixture_server: Fixture rows = client.congressional.profiles() assert rows[0].image is None + + +def test_net_worth_decodes_fractional_value_and_null_members(client: Any, fixture_server: FixtureServer) -> None: + """A fractional ``value`` decodes exactly; null itemized members and a null range ``max`` decode to ``None``.""" + body = load_fixture("congress_senate_net_worth.json") + body[0]["value"] = 32500.5 + del body[0]["debtDetails"]["dateIncurred"] + fractional = dict(body[0], category=None, name=None, owner=None, value=None, valueRange={"min": 1, "max": None}) + fixture_server.route("/senate-net-worth", [body[0], fractional]) + rows = client.congressional.net_worth(MEMBER_ID) + + assert rows[0].value == 32500.5 + assert rows[0].debt_details.date_incurred is None + assert rows[1].category is None + assert rows[1].name is None + assert rows[1].owner is None + assert rows[1].value is None + assert rows[1].value_range.max is None + + +def test_net_worth_aggregated_decodes_fractional_amounts_and_absent_members( + client: Any, fixture_server: FixtureServer +) -> None: + """Fractional amounts decode exactly and absent aggregate members decode to ``None``.""" + body = load_fixture("congress_senate_net_worth_aggregated.json") + for member in ( + "realEstateLiabilities", + "businessAndSelfEmployment", + "ownershipInterest", + "options", + "revolvingAndCreditLines", + "assetBackedSecurities", + "businessLiabilities", + ): + del body[0][member] + body[0].update(total=59082540.5, cashAndCashEquivalents=121004.5, realEstate=3000000.5, stock=8000.5) + body[0]["mutualFundsAndETFs"] = 34526531.5 + sparse = {key: value for key, value in body[0].items() if key not in ("realEstate", "stock")} + fixture_server.route("/senate-net-worth-aggregated", [body[0], sparse]) + rows = client.congressional.net_worth_aggregated(MEMBER_ID) + row = rows[0] + + assert row.total == 59082540.5 + assert row.cash_and_cash_equivalents == 121004.5 + assert row.real_estate == 3000000.5 + assert row.stock == 8000.5 + assert row.mutual_funds_and_etfs == 34526531.5 + assert row.real_estate_liabilities is None + assert row.business_and_self_employment is None + assert row.ownership_interest is None + assert row.options is None + assert row.revolving_and_credit_lines is None + assert row.asset_backed_securities is None + assert row.business_liabilities is None + assert rows[1].real_estate is None + assert rows[1].stock is None diff --git a/crates/fmp-py/tests/typecheck_contract.py b/crates/fmp-py/tests/typecheck_contract.py index a175e66..1473b11 100644 --- a/crates/fmp-py/tests/typecheck_contract.py +++ b/crates/fmp-py/tests/typecheck_contract.py @@ -206,8 +206,10 @@ def check_congressional_contract(client: FmpClient) -> None: totals: list[CongressionalMemberNetWorthAggregate] = client.congressional.net_worth_aggregated( "P000197", totals_col="stock" ) - total: int = totals[0].total - _ = (disclosed, gains, born, years_active, value_range, total) + value: float | None = entries[0].value + total: float = totals[0].total + options: float | None = totals[0].options + _ = (disclosed, gains, born, years_active, value_range, value, total, options) def check_funds_contract(client: FmpClient) -> None: diff --git a/crates/libfmp/src/responses/congressional.rs b/crates/libfmp/src/responses/congressional.rs index 8a193df..07c0c53 100644 --- a/crates/libfmp/src/responses/congressional.rs +++ b/crates/libfmp/src/responses/congressional.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Deserializer, Serialize}; use crate::{ codecs::{DynamicJson, OpaqueDateText, TitleCaseBoolFlag}, - types::{CalendarYear, CongressionalMemberId, Date, FormType}, + types::{CalendarYear, CongressionalMemberId, Date, FormType, MarketValue}, }; fn required_option<'de, D, T>(deserializer: D) -> Result, D::Error> @@ -94,7 +94,8 @@ pub struct CongressionalMemberPosition { #[serde(rename_all = "camelCase")] pub struct CongressionalNetWorthRange { pub min: i64, - pub max: i64, + #[serde(deserialize_with = "required_option")] + pub max: Option, } /// Opaque provider details for a disclosed debt. @@ -102,11 +103,12 @@ pub struct CongressionalNetWorthRange { #[non_exhaustive] #[serde(rename_all = "camelCase")] pub struct CongressionalDebtDetails { - pub date_incurred: OpaqueDateText, + #[serde(skip_serializing_if = "Option::is_none")] + pub date_incurred: Option, } /// One itemized congressional net-worth disclosure. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[non_exhaustive] #[serde(rename_all = "camelCase")] pub struct CongressionalMemberNetWorth { @@ -116,19 +118,24 @@ pub struct CongressionalMemberNetWorth { pub year: CalendarYear, pub filing_date: Date, pub section: String, - pub category: String, - pub name: String, + #[serde(deserialize_with = "required_option")] + pub category: Option, + #[serde(deserialize_with = "required_option")] + pub name: Option, pub asset_type: String, #[serde(deserialize_with = "required_option")] pub income_type: Option, - pub owner: String, + #[serde(deserialize_with = "required_option")] + pub owner: Option, #[serde(deserialize_with = "required_option")] pub comment: Option, #[serde(deserialize_with = "required_option")] pub debt_details: Option, #[serde(deserialize_with = "required_option")] pub value_range: Option, - pub value: i64, + #[serde(deserialize_with = "required_option")] + #[serde(serialize_with = "crate::codecs::integral_f64::serialize_option")] + pub value: Option, #[serde(deserialize_with = "required_option")] pub income_range: Option, #[serde(deserialize_with = "required_option")] @@ -137,24 +144,48 @@ pub struct CongressionalMemberNetWorth { } /// Aggregated congressional net-worth totals for one filing year. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +/// +/// Only the member, year, total, cash and mutual-fund members are sent on +/// every row; the other amounts are `None` when the key is absent. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[non_exhaustive] #[serde(rename_all = "camelCase")] pub struct CongressionalMemberNetWorthAggregate { #[serde(rename = "senateID")] pub member_id: CongressionalMemberId, pub year: CalendarYear, - pub total: i64, - pub real_estate_liabilities: i64, - pub cash_and_cash_equivalents: i64, - pub business_and_self_employment: i64, - pub real_estate: i64, - pub ownership_interest: i64, - pub stock: i64, - pub options: i64, - pub revolving_and_credit_lines: i64, - pub asset_backed_securities: i64, - pub business_liabilities: i64, + #[serde(serialize_with = "crate::codecs::integral_f64::serialize")] + pub total: MarketValue, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(serialize_with = "crate::codecs::integral_f64::serialize_option")] + pub real_estate_liabilities: Option, + #[serde(serialize_with = "crate::codecs::integral_f64::serialize")] + pub cash_and_cash_equivalents: MarketValue, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(serialize_with = "crate::codecs::integral_f64::serialize_option")] + pub business_and_self_employment: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(serialize_with = "crate::codecs::integral_f64::serialize_option")] + pub real_estate: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(serialize_with = "crate::codecs::integral_f64::serialize_option")] + pub ownership_interest: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(serialize_with = "crate::codecs::integral_f64::serialize_option")] + pub stock: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(serialize_with = "crate::codecs::integral_f64::serialize_option")] + pub options: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(serialize_with = "crate::codecs::integral_f64::serialize_option")] + pub revolving_and_credit_lines: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(serialize_with = "crate::codecs::integral_f64::serialize_option")] + pub asset_backed_securities: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(serialize_with = "crate::codecs::integral_f64::serialize_option")] + pub business_liabilities: Option, #[serde(rename = "mutualFundsAndETFs")] - pub mutual_funds_and_etfs: i64, + #[serde(serialize_with = "crate::codecs::integral_f64::serialize")] + pub mutual_funds_and_etfs: MarketValue, } diff --git a/crates/libfmp/tests/congressional_net_worth_endpoints.rs b/crates/libfmp/tests/congressional_net_worth_endpoints.rs index 913f4ab..b0efb5c 100644 --- a/crates/libfmp/tests/congressional_net_worth_endpoints.rs +++ b/crates/libfmp/tests/congressional_net_worth_endpoints.rs @@ -109,11 +109,18 @@ async fn custom_proxy_preserves_queries_headers_bare_arrays_and_exact_fixtures() assert_eq!(net_worth[0].member_id.as_str(), "P000197"); assert_eq!( - net_worth[0].debt_details.as_ref().unwrap().date_incurred.0, + net_worth[0] + .debt_details + .as_ref() + .unwrap() + .date_incurred + .as_ref() + .unwrap() + .0, "September 2007" ); assert_eq!(net_worth[0].income, None); - assert_eq!(aggregated[0].total, 225_219_551); + assert_eq!(aggregated[0].total, 225_219_551.0); assert_eq!( serde_json::to_value(&net_worth).unwrap(), serde_json::from_slice::(NET_WORTH).unwrap() @@ -172,14 +179,14 @@ async fn optional_query_values_stay_omitted_and_unknown_response_fields_are_tole let member_id = || CongressionalMemberId::new("P000197").unwrap(); let rows = client.congressional_net_worth(member_id()).await.unwrap(); - assert_eq!(rows[0].value, -1); + assert_eq!(rows[0].value, Some(-1.0)); assert!(rows[0].debt_details.is_none()); assert_eq!(rows[0].income_range.as_ref().unwrap().min, -10); let totals = client .congressional_net_worth_aggregated(member_id()) .await .unwrap(); - assert_eq!(totals[0].total, -1); + assert_eq!(totals[0].total, -1.0); assert_eq!( executor @@ -225,10 +232,14 @@ fn itemized_required_fields_and_nullable_shapes_are_strict() { } for field in [ + "category", + "name", "incomeType", + "owner", "comment", "debtDetails", "valueRange", + "value", "incomeRange", "income", ] { @@ -246,11 +257,7 @@ fn itemized_required_fields_and_nullable_shapes_are_strict() { "year", "filingDate", "section", - "category", - "name", "assetType", - "owner", - "value", "link", ] { let mut invalid = row.clone(); @@ -263,24 +270,67 @@ fn itemized_required_fields_and_nullable_shapes_are_strict() { } #[test] -fn every_aggregated_field_is_required_and_typed() { +fn itemized_fractional_value_and_null_members_decode_exactly() { + let mut row = serde_json::from_slice::(NET_WORTH).unwrap()[0].clone(); + row["value"] = serde_json::json!(32500.5); + row["debtDetails"] = serde_json::json!({}); + let entry = serde_json::from_value::(row.clone()).unwrap(); + assert_eq!(entry.value, Some(32500.5)); + assert!(entry.debt_details.as_ref().unwrap().date_incurred.is_none()); + assert_eq!(serde_json::to_value(&entry).unwrap(), row); + + row["valueRange"]["max"] = serde_json::Value::Null; + for field in ["category", "name", "owner", "value"] { + row[field] = serde_json::Value::Null; + } + let entry = serde_json::from_value::(row).unwrap(); + assert_eq!(entry.category, None); + assert_eq!(entry.name, None); + assert_eq!(entry.owner, None); + assert_eq!(entry.value, None); + assert_eq!(entry.value_range.unwrap().max, None); +} + +const AGGREGATE_REQUIRED: [&str; 5] = [ + "senateID", + "year", + "total", + "cashAndCashEquivalents", + "mutualFundsAndETFs", +]; + +const AGGREGATE_OMITTABLE: [&str; 9] = [ + "realEstate", + "stock", + "realEstateLiabilities", + "businessAndSelfEmployment", + "ownershipInterest", + "options", + "revolvingAndCreditLines", + "assetBackedSecurities", + "businessLiabilities", +]; + +#[test] +fn aggregated_required_fields_are_strict_and_omittable_fields_may_be_absent() { let row = serde_json::from_slice::(AGGREGATED).unwrap()[0].clone(); - for field in [ - "senateID", - "year", - "total", - "realEstateLiabilities", - "cashAndCashEquivalents", - "businessAndSelfEmployment", - "realEstate", - "ownershipInterest", - "stock", - "options", - "revolvingAndCreditLines", - "assetBackedSecurities", - "businessLiabilities", - "mutualFundsAndETFs", - ] { + for field in AGGREGATE_REQUIRED { + let mut missing = row.clone(); + missing.as_object_mut().unwrap().remove(field); + assert!( + serde_json::from_value::(missing).is_err(), + "aggregate field {field} must be present" + ); + } + for field in AGGREGATE_REQUIRED.iter().chain(&AGGREGATE_OMITTABLE) { + let mut invalid = row.clone(); + invalid[field] = serde_json::json!(true); + assert!( + serde_json::from_value::(invalid).is_err(), + "aggregate field {field} must be typed" + ); + } + for field in AGGREGATE_REQUIRED { let mut invalid = row.clone(); invalid[field] = serde_json::Value::Null; assert!( @@ -288,6 +338,52 @@ fn every_aggregated_field_is_required_and_typed() { "aggregate field {field} must not accept null" ); } + for field in AGGREGATE_OMITTABLE { + let mut nulled = row.clone(); + nulled[field] = serde_json::Value::Null; + let totals = + serde_json::from_value::(nulled).unwrap(); + let encoded = serde_json::to_value(&totals).unwrap(); + assert!( + encoded.get(field).is_none(), + "aggregate field {field} must decode null as None" + ); + } + + let mut sparse = row.clone(); + for field in AGGREGATE_OMITTABLE { + sparse.as_object_mut().unwrap().remove(field); + } + sparse["total"] = serde_json::json!(59082540.5); + sparse["cashAndCashEquivalents"] = serde_json::json!(121004.5); + sparse["mutualFundsAndETFs"] = serde_json::json!(34526531.5); + let totals = + serde_json::from_value::(sparse.clone()).unwrap(); + assert_eq!(totals.total, 59_082_540.5); + assert_eq!(totals.cash_and_cash_equivalents, 121_004.5); + assert_eq!(totals.mutual_funds_and_etfs, 34_526_531.5); + assert_eq!(totals.real_estate, None); + assert_eq!(totals.stock, None); + assert_eq!(totals.real_estate_liabilities, None); + assert_eq!(totals.business_and_self_employment, None); + assert_eq!(totals.ownership_interest, None); + assert_eq!(totals.options, None); + assert_eq!(totals.revolving_and_credit_lines, None); + assert_eq!(totals.asset_backed_securities, None); + assert_eq!(totals.business_liabilities, None); + assert_eq!(serde_json::to_value(&totals).unwrap(), sparse); + + let totals = serde_json::from_value::(row).unwrap(); + assert_eq!(totals.asset_backed_securities, Some(4_475_006.0)); + assert_eq!(totals.options, Some(0.0)); + + let mut fractional = sparse; + fractional["realEstate"] = serde_json::json!(3000000.5); + fractional["stock"] = serde_json::json!(8000.5); + let totals = + serde_json::from_value::(fractional).unwrap(); + assert_eq!(totals.real_estate, Some(3_000_000.5)); + assert_eq!(totals.stock, Some(8_000.5)); } #[tokio::test] diff --git a/docs/adr/0033-nullable-response-members.md b/docs/adr/0033-nullable-response-members.md index 61abbf8..ae5c28a 100644 --- a/docs/adr/0033-nullable-response-members.md +++ b/docs/adr/0033-nullable-response-members.md @@ -29,6 +29,7 @@ evidence is now a second source, so both need a rule. | `""` on a date | `Option` through `codecs::empty_or_null_date` | `None` | `nil` | | `""` on a plain `String` | unchanged: stays `""` | `""` | `""` | | `"NULL"` text on a `NumericString` (`FundDisclosureSearchResult.entityOrgType`, #380) | `Option` through `codecs::null_text` | `None` | `nil` | +| key absent from the object | plain `Option` with `skip_serializing_if = "Option::is_none"` (no `required_option`) | `None` | `*T` with `omitzero` | - Evidence decides: a member changes only when the audit (or a later report) saw it null or empty. Parity siblings are not widened. @@ -44,6 +45,20 @@ evidence is now a second source, so both need a rule. used only where the evidence shows `"NULL"` standing in for an absent value (the live rows that send it on `entityOrgType` also send a null `address`); no other codec treats `"NULL"` as absent. +- A member is optional-when-absent only when the audit saw its key + missing, not merely null; every other member keeps a required key. + First case, [#377](https://github.com/bitbrew-dev/libfmp/issues/377): + `CongressionalMemberNetWorthAggregate` omits `assetBackedSecurities`, + `businessAndSelfEmployment`, `businessLiabilities`, `options`, + `ownershipInterest`, `realEstateLiabilities` and + `revolvingAndCreditLines` on all 11 audited `senate-net-worth-aggregated` + rows, and a 14-member live probe (131 rows) also saw `realEstate` and + `stock` absent; only `senateID`, `year`, `total`, + `cashAndCashEquivalents` and `mutualFundsAndETFs` were always sent. + `CongressionalDebtDetails` omits `dateIncurred` on 30 audited + `senate-net-worth` rows. The same audit saw fractional amounts + (`32500.5`) on members typed as integers; those became `MarketValue` + (`f64`) under the next rule. - A member whose non-null value has the wrong type (a year where a date is documented) is a separate type fix, not an `Option`. - Every change is breaking and ships in a minor release with a @@ -57,6 +72,9 @@ shadow member keeps the raw `jsontext.Value`, the key is required, and `""` or null decode to `nil`. `crate::codecs::null_text::deserialize` maps the same way to `Codec::NullTextString`, where the text `"NULL"` or null decode to `nil`. It fails on any other shape rather than guessing. +A plain `Option` without a codec is `Codec::Plain`: the key +may be absent or null, and `skip_serializing_if = "Option::is_none"` adds +`omitzero` so a re-encoded row omits it as serde does. ## Alternatives considered diff --git a/sdk/go/congressional_models.go b/sdk/go/congressional_models.go index 2afad26..ac827cc 100644 --- a/sdk/go/congressional_models.go +++ b/sdk/go/congressional_models.go @@ -273,16 +273,16 @@ func (m *CongressionalMemberPosition) UnmarshalJSONFrom(dec *jsontext.Decoder) e // CongressionalNetWorthRange is a minimum and maximum value disclosed for a // congressional asset or income. type CongressionalNetWorthRange struct { - Min int64 `json:"min"` - Max int64 `json:"max"` + Min int64 `json:"min"` + Max *int64 `json:"max"` } // congressionalNetWorthRangeShadow mirrors CongressionalNetWorthRange with a // pointer or raw value for every required member so a missing or null member // is observable after decoding. type congressionalNetWorthRangeShadow struct { - Min *int64 `json:"min"` - Max *int64 `json:"max"` + Min *int64 `json:"min"` + Max jsontext.Value `json:"max"` } // UnmarshalJSONFrom decodes one JSON object and rejects it with a Decode @@ -296,19 +296,27 @@ func (m *CongressionalNetWorthRange) UnmarshalJSONFrom(dec *jsontext.Decoder) er switch { case shadow.Min == nil: return missingMemberError("CongressionalNetWorthRange", "min") - case shadow.Max == nil: + case len(shadow.Max) == 0: return missingMemberError("CongressionalNetWorthRange", "max") } + var max *int64 + if shadow.Max.Kind() != 'n' { + var value int64 + if err := json.Unmarshal(shadow.Max, &value); err != nil { + return err + } + max = &value + } *m = CongressionalNetWorthRange{ Min: *shadow.Min, - Max: *shadow.Max, + Max: max, } return nil } // CongressionalDebtDetails is opaque provider details for a disclosed debt. type CongressionalDebtDetails struct { - DateIncurred string `json:"dateIncurred"` + DateIncurred *string `json:"dateIncurred,omitzero"` } // congressionalDebtDetailsShadow mirrors CongressionalDebtDetails with a @@ -326,12 +334,8 @@ func (m *CongressionalDebtDetails) UnmarshalJSONFrom(dec *jsontext.Decoder) erro if err := json.UnmarshalDecode(dec, &shadow); err != nil { return err } - switch { - case shadow.DateIncurred == nil: - return missingMemberError("CongressionalDebtDetails", "dateIncurred") - } *m = CongressionalDebtDetails{ - DateIncurred: *shadow.DateIncurred, + DateIncurred: shadow.DateIncurred, } return nil } @@ -344,15 +348,15 @@ type CongressionalMemberNetWorth struct { Year uint32 `json:"year"` FilingDate Date `json:"filingDate"` Section string `json:"section"` - Category string `json:"category"` - Name string `json:"name"` + Category *string `json:"category"` + Name *string `json:"name"` AssetType string `json:"assetType"` IncomeType *string `json:"incomeType"` - Owner string `json:"owner"` + Owner *string `json:"owner"` Comment *string `json:"comment"` DebtDetails *CongressionalDebtDetails `json:"debtDetails"` ValueRange *CongressionalNetWorthRange `json:"valueRange"` - Value int64 `json:"value"` + Value *float64 `json:"value"` IncomeRange *CongressionalNetWorthRange `json:"incomeRange"` Income *jsontext.Value `json:"income"` Link string `json:"link"` @@ -367,15 +371,15 @@ type congressionalMemberNetWorthShadow struct { Year *uint32 `json:"year"` FilingDate *Date `json:"filingDate"` Section *string `json:"section"` - Category *string `json:"category"` - Name *string `json:"name"` + Category jsontext.Value `json:"category"` + Name jsontext.Value `json:"name"` AssetType *string `json:"assetType"` IncomeType jsontext.Value `json:"incomeType"` - Owner *string `json:"owner"` + Owner jsontext.Value `json:"owner"` Comment jsontext.Value `json:"comment"` DebtDetails jsontext.Value `json:"debtDetails"` ValueRange jsontext.Value `json:"valueRange"` - Value *int64 `json:"value"` + Value jsontext.Value `json:"value"` IncomeRange jsontext.Value `json:"incomeRange"` Income jsontext.Value `json:"income"` Link *string `json:"link"` @@ -400,15 +404,15 @@ func (m *CongressionalMemberNetWorth) UnmarshalJSONFrom(dec *jsontext.Decoder) e return missingMemberError("CongressionalMemberNetWorth", "filingDate") case shadow.Section == nil: return missingMemberError("CongressionalMemberNetWorth", "section") - case shadow.Category == nil: + case len(shadow.Category) == 0: return missingMemberError("CongressionalMemberNetWorth", "category") - case shadow.Name == nil: + case len(shadow.Name) == 0: return missingMemberError("CongressionalMemberNetWorth", "name") case shadow.AssetType == nil: return missingMemberError("CongressionalMemberNetWorth", "assetType") case len(shadow.IncomeType) == 0: return missingMemberError("CongressionalMemberNetWorth", "incomeType") - case shadow.Owner == nil: + case len(shadow.Owner) == 0: return missingMemberError("CongressionalMemberNetWorth", "owner") case len(shadow.Comment) == 0: return missingMemberError("CongressionalMemberNetWorth", "comment") @@ -416,7 +420,7 @@ func (m *CongressionalMemberNetWorth) UnmarshalJSONFrom(dec *jsontext.Decoder) e return missingMemberError("CongressionalMemberNetWorth", "debtDetails") case len(shadow.ValueRange) == 0: return missingMemberError("CongressionalMemberNetWorth", "valueRange") - case shadow.Value == nil: + case len(shadow.Value) == 0: return missingMemberError("CongressionalMemberNetWorth", "value") case len(shadow.IncomeRange) == 0: return missingMemberError("CongressionalMemberNetWorth", "incomeRange") @@ -425,6 +429,22 @@ func (m *CongressionalMemberNetWorth) UnmarshalJSONFrom(dec *jsontext.Decoder) e case shadow.Link == nil: return missingMemberError("CongressionalMemberNetWorth", "link") } + var category *string + if shadow.Category.Kind() != 'n' { + var value string + if err := json.Unmarshal(shadow.Category, &value); err != nil { + return err + } + category = &value + } + var name *string + if shadow.Name.Kind() != 'n' { + var value string + if err := json.Unmarshal(shadow.Name, &value); err != nil { + return err + } + name = &value + } var incomeType *string if shadow.IncomeType.Kind() != 'n' { var value string @@ -433,6 +453,14 @@ func (m *CongressionalMemberNetWorth) UnmarshalJSONFrom(dec *jsontext.Decoder) e } incomeType = &value } + var owner *string + if shadow.Owner.Kind() != 'n' { + var value string + if err := json.Unmarshal(shadow.Owner, &value); err != nil { + return err + } + owner = &value + } var comment *string if shadow.Comment.Kind() != 'n' { var value string @@ -457,6 +485,14 @@ func (m *CongressionalMemberNetWorth) UnmarshalJSONFrom(dec *jsontext.Decoder) e } valueRange = &value } + var valueMember *float64 + if shadow.Value.Kind() != 'n' { + var value float64 + if err := json.Unmarshal(shadow.Value, &value); err != nil { + return err + } + valueMember = &value + } var incomeRange *CongressionalNetWorthRange if shadow.IncomeRange.Kind() != 'n' { var value CongressionalNetWorthRange @@ -479,15 +515,15 @@ func (m *CongressionalMemberNetWorth) UnmarshalJSONFrom(dec *jsontext.Decoder) e Year: *shadow.Year, FilingDate: *shadow.FilingDate, Section: *shadow.Section, - Category: *shadow.Category, - Name: *shadow.Name, + Category: category, + Name: name, AssetType: *shadow.AssetType, IncomeType: incomeType, - Owner: *shadow.Owner, + Owner: owner, Comment: comment, DebtDetails: debtDetails, ValueRange: valueRange, - Value: *shadow.Value, + Value: valueMember, IncomeRange: incomeRange, Income: income, Link: *shadow.Link, @@ -498,40 +534,40 @@ func (m *CongressionalMemberNetWorth) UnmarshalJSONFrom(dec *jsontext.Decoder) e // CongressionalMemberNetWorthAggregate is aggregated congressional net-worth // totals for one filing year. type CongressionalMemberNetWorthAggregate struct { - MemberID string `json:"senateID"` - Year uint32 `json:"year"` - Total int64 `json:"total"` - RealEstateLiabilities int64 `json:"realEstateLiabilities"` - CashAndCashEquivalents int64 `json:"cashAndCashEquivalents"` - BusinessAndSelfEmployment int64 `json:"businessAndSelfEmployment"` - RealEstate int64 `json:"realEstate"` - OwnershipInterest int64 `json:"ownershipInterest"` - Stock int64 `json:"stock"` - Options int64 `json:"options"` - RevolvingAndCreditLines int64 `json:"revolvingAndCreditLines"` - AssetBackedSecurities int64 `json:"assetBackedSecurities"` - BusinessLiabilities int64 `json:"businessLiabilities"` - MutualFundsAndETFs int64 `json:"mutualFundsAndETFs"` + MemberID string `json:"senateID"` + Year uint32 `json:"year"` + Total float64 `json:"total"` + RealEstateLiabilities *float64 `json:"realEstateLiabilities,omitzero"` + CashAndCashEquivalents float64 `json:"cashAndCashEquivalents"` + BusinessAndSelfEmployment *float64 `json:"businessAndSelfEmployment,omitzero"` + RealEstate *float64 `json:"realEstate,omitzero"` + OwnershipInterest *float64 `json:"ownershipInterest,omitzero"` + Stock *float64 `json:"stock,omitzero"` + Options *float64 `json:"options,omitzero"` + RevolvingAndCreditLines *float64 `json:"revolvingAndCreditLines,omitzero"` + AssetBackedSecurities *float64 `json:"assetBackedSecurities,omitzero"` + BusinessLiabilities *float64 `json:"businessLiabilities,omitzero"` + MutualFundsAndETFs float64 `json:"mutualFundsAndETFs"` } // congressionalMemberNetWorthAggregateShadow mirrors // CongressionalMemberNetWorthAggregate with a pointer or raw value for every // required member so a missing or null member is observable after decoding. type congressionalMemberNetWorthAggregateShadow struct { - MemberID *string `json:"senateID"` - Year *uint32 `json:"year"` - Total *int64 `json:"total"` - RealEstateLiabilities *int64 `json:"realEstateLiabilities"` - CashAndCashEquivalents *int64 `json:"cashAndCashEquivalents"` - BusinessAndSelfEmployment *int64 `json:"businessAndSelfEmployment"` - RealEstate *int64 `json:"realEstate"` - OwnershipInterest *int64 `json:"ownershipInterest"` - Stock *int64 `json:"stock"` - Options *int64 `json:"options"` - RevolvingAndCreditLines *int64 `json:"revolvingAndCreditLines"` - AssetBackedSecurities *int64 `json:"assetBackedSecurities"` - BusinessLiabilities *int64 `json:"businessLiabilities"` - MutualFundsAndETFs *int64 `json:"mutualFundsAndETFs"` + MemberID *string `json:"senateID"` + Year *uint32 `json:"year"` + Total *float64 `json:"total"` + RealEstateLiabilities *float64 `json:"realEstateLiabilities"` + CashAndCashEquivalents *float64 `json:"cashAndCashEquivalents"` + BusinessAndSelfEmployment *float64 `json:"businessAndSelfEmployment"` + RealEstate *float64 `json:"realEstate"` + OwnershipInterest *float64 `json:"ownershipInterest"` + Stock *float64 `json:"stock"` + Options *float64 `json:"options"` + RevolvingAndCreditLines *float64 `json:"revolvingAndCreditLines"` + AssetBackedSecurities *float64 `json:"assetBackedSecurities"` + BusinessLiabilities *float64 `json:"businessLiabilities"` + MutualFundsAndETFs *float64 `json:"mutualFundsAndETFs"` } // UnmarshalJSONFrom decodes one JSON object and rejects it with a Decode @@ -549,26 +585,8 @@ func (m *CongressionalMemberNetWorthAggregate) UnmarshalJSONFrom(dec *jsontext.D return missingMemberError("CongressionalMemberNetWorthAggregate", "year") case shadow.Total == nil: return missingMemberError("CongressionalMemberNetWorthAggregate", "total") - case shadow.RealEstateLiabilities == nil: - return missingMemberError("CongressionalMemberNetWorthAggregate", "realEstateLiabilities") case shadow.CashAndCashEquivalents == nil: return missingMemberError("CongressionalMemberNetWorthAggregate", "cashAndCashEquivalents") - case shadow.BusinessAndSelfEmployment == nil: - return missingMemberError("CongressionalMemberNetWorthAggregate", "businessAndSelfEmployment") - case shadow.RealEstate == nil: - return missingMemberError("CongressionalMemberNetWorthAggregate", "realEstate") - case shadow.OwnershipInterest == nil: - return missingMemberError("CongressionalMemberNetWorthAggregate", "ownershipInterest") - case shadow.Stock == nil: - return missingMemberError("CongressionalMemberNetWorthAggregate", "stock") - case shadow.Options == nil: - return missingMemberError("CongressionalMemberNetWorthAggregate", "options") - case shadow.RevolvingAndCreditLines == nil: - return missingMemberError("CongressionalMemberNetWorthAggregate", "revolvingAndCreditLines") - case shadow.AssetBackedSecurities == nil: - return missingMemberError("CongressionalMemberNetWorthAggregate", "assetBackedSecurities") - case shadow.BusinessLiabilities == nil: - return missingMemberError("CongressionalMemberNetWorthAggregate", "businessLiabilities") case shadow.MutualFundsAndETFs == nil: return missingMemberError("CongressionalMemberNetWorthAggregate", "mutualFundsAndETFs") } @@ -576,16 +594,16 @@ func (m *CongressionalMemberNetWorthAggregate) UnmarshalJSONFrom(dec *jsontext.D MemberID: *shadow.MemberID, Year: *shadow.Year, Total: *shadow.Total, - RealEstateLiabilities: *shadow.RealEstateLiabilities, + RealEstateLiabilities: shadow.RealEstateLiabilities, CashAndCashEquivalents: *shadow.CashAndCashEquivalents, - BusinessAndSelfEmployment: *shadow.BusinessAndSelfEmployment, - RealEstate: *shadow.RealEstate, - OwnershipInterest: *shadow.OwnershipInterest, - Stock: *shadow.Stock, - Options: *shadow.Options, - RevolvingAndCreditLines: *shadow.RevolvingAndCreditLines, - AssetBackedSecurities: *shadow.AssetBackedSecurities, - BusinessLiabilities: *shadow.BusinessLiabilities, + BusinessAndSelfEmployment: shadow.BusinessAndSelfEmployment, + RealEstate: shadow.RealEstate, + OwnershipInterest: shadow.OwnershipInterest, + Stock: shadow.Stock, + Options: shadow.Options, + RevolvingAndCreditLines: shadow.RevolvingAndCreditLines, + AssetBackedSecurities: shadow.AssetBackedSecurities, + BusinessLiabilities: shadow.BusinessLiabilities, MutualFundsAndETFs: *shadow.MutualFundsAndETFs, } return nil diff --git a/sdk/go/congressional_parity_test.go b/sdk/go/congressional_parity_test.go index ac9d410..967bea7 100644 --- a/sdk/go/congressional_parity_test.go +++ b/sdk/go/congressional_parity_test.go @@ -123,28 +123,29 @@ func TestDocumentedMemberAndNetWorthFixturesDecodeExactValues(t *testing.T) { entry := entries[0] if entry.MemberID != "P000197" || entry.FormType != "House Report" || entry.Year != 2022 || entry.FilingDate != mustParseDate(t, "2023-05-15") || entry.Section != "Liabilities" || - entry.Category != "Mortgage & Real Estate Liability" || entry.Name != "Union Bank of California" || - entry.AssetType != "Mortgage on 2640 Broadway, San Francisco, CA" || entry.Owner != "Joint" || - entry.Value != 3_000_001 || + !reflect.DeepEqual(entry.Category, new("Mortgage & Real Estate Liability")) || + !reflect.DeepEqual(entry.Name, new("Union Bank of California")) || + entry.AssetType != "Mortgage on 2640 Broadway, San Francisco, CA" || !reflect.DeepEqual(entry.Owner, new("Joint")) || + !reflect.DeepEqual(entry.Value, new(3_000_001.0)) || entry.Link != "https://disclosures-clerk.house.gov/public_disc/financial-pdfs/2022/10053231.pdf" { t.Fatalf("congress_senate_net_worth = %+v", entry) } if entry.IncomeType != nil || entry.Comment != nil || entry.IncomeRange != nil || entry.Income != nil { t.Fatalf("null members decoded as non-nil: %+v", entry) } - if entry.DebtDetails == nil || *entry.DebtDetails != (CongressionalDebtDetails{DateIncurred: "September 2007"}) { + if !reflect.DeepEqual(entry.DebtDetails, &CongressionalDebtDetails{DateIncurred: new("September 2007")}) { t.Fatalf("debtDetails = %+v", entry.DebtDetails) } - if entry.ValueRange == nil || *entry.ValueRange != (CongressionalNetWorthRange{Min: 1_000_001, Max: 5_000_000}) { + if !reflect.DeepEqual(entry.ValueRange, &CongressionalNetWorthRange{Min: 1_000_001, Max: new(int64(5_000_000))}) { t.Fatalf("valueRange = %+v", entry.ValueRange) } totals := assertFixtureParity[CongressionalMemberNetWorthAggregate](t, "congress_senate_net_worth_aggregated.json") if want := (CongressionalMemberNetWorthAggregate{MemberID: "P000197", Year: 2024, Total: 225_219_551, - RealEstateLiabilities: 27_000_005, CashAndCashEquivalents: 291_009, BusinessAndSelfEmployment: 0, - RealEstate: 45_032_504, OwnershipInterest: 70_140_014, Stock: 136_748_525, Options: 0, - RevolvingAndCreditLines: 1_500_002, AssetBackedSecurities: 4_475_006, BusinessLiabilities: 3_000_001, - MutualFundsAndETFs: 32_501}); len(totals) != 1 || totals[0] != want { + RealEstateLiabilities: new(27_000_005.0), CashAndCashEquivalents: 291_009, BusinessAndSelfEmployment: new(0.0), + RealEstate: new(45_032_504.0), OwnershipInterest: new(70_140_014.0), Stock: new(136_748_525.0), Options: new(0.0), + RevolvingAndCreditLines: new(1_500_002.0), AssetBackedSecurities: new(4_475_006.0), BusinessLiabilities: new(3_000_001.0), + MutualFundsAndETFs: 32_501}); len(totals) != 1 || !reflect.DeepEqual(totals[0], want) { t.Fatalf("congress_senate_net_worth_aggregated = %+v", totals) } if encoded, err := json.Marshal(totals[0]); err != nil || !strings.Contains(string(encoded), `"mutualFundsAndETFs":32501`) { @@ -174,7 +175,6 @@ func TestCongressionalRequiredMembersAreEnforcedLikeSerde(t *testing.T) { {"position with null senateID", `[` + strings.Replace(position, `"Z000018"`, `null`, 1) + `,"endDate":null}]`, "CongressionalMemberPosition", "senateID"}, {"entry without debtDetails", `[` + entry + `}]`, "CongressionalMemberNetWorth", "debtDetails"}, - {"entry with empty debtDetails", `[` + entry + `,"debtDetails":{}}]`, "CongressionalDebtDetails", "dateIncurred"}, {"entry with half a range", `[` + strings.Replace(entry, `"valueRange":null`, `"valueRange":{"min":1}`, 1) + `,"debtDetails":null}]`, "CongressionalNetWorthRange", "max"}, @@ -206,7 +206,7 @@ func TestCongressionalRequiredMembersAreEnforcedLikeSerde(t *testing.T) { } var entries []CongressionalMemberNetWorth if err := json.Unmarshal([]byte(`[`+entry+`,"debtDetails":null}]`), &entries); err != nil || len(entries) != 1 || - entries[0].DebtDetails != nil || entries[0].Value != -1 || entries[0].IncomeRange == nil || + entries[0].DebtDetails != nil || !reflect.DeepEqual(entries[0].Value, new(-1.0)) || entries[0].IncomeRange == nil || entries[0].IncomeRange.Min != -10 { t.Fatalf("entry with null debtDetails = %+v, %v", entries, err) } @@ -254,3 +254,79 @@ func TestCongressionalMemberProfileImageDecodesNullAsNil(t *testing.T) { }) } } + +// Mirrors itemized_fractional_value_and_null_members_decode_exactly and +// aggregated_required_fields_are_strict_and_omittable_fields_may_be_absent in +// crates/libfmp/tests/congressional_net_worth_endpoints.rs: fractional amounts +// decode exactly, null itemized members and absent aggregate members decode to +// nil, and an empty debtDetails object has no dateIncurred. +func TestCongressionalNetWorthFractionalNullAndAbsentMembers(t *testing.T) { + t.Parallel() + const netWorth = "congress_senate_net_worth.json" + var entries []CongressionalMemberNetWorth + fractional := mutateFixtureMember(t, netWorth, "value", jsontext.Value(`32500.5`)) + if err := json.Unmarshal(fractional, &entries); err != nil || len(entries) != 1 || + !reflect.DeepEqual(entries[0].Value, new(32500.5)) { + t.Fatalf("fractional value = %+v, %v", entries, err) + } + nullable := []struct { + member string + isNil func(CongressionalMemberNetWorth) bool + }{ + {"category", func(row CongressionalMemberNetWorth) bool { return row.Category == nil }}, + {"name", func(row CongressionalMemberNetWorth) bool { return row.Name == nil }}, + {"owner", func(row CongressionalMemberNetWorth) bool { return row.Owner == nil }}, + {"value", func(row CongressionalMemberNetWorth) bool { return row.Value == nil }}, + {"valueRange.max", func(row CongressionalMemberNetWorth) bool { + return row.ValueRange != nil && row.ValueRange.Max == nil + }}, + } + for _, tc := range nullable { + var rows []CongressionalMemberNetWorth + if err := json.Unmarshal(mutateFixtureMember(t, netWorth, tc.member, jsontext.Value(`null`)), &rows); err != nil || + len(rows) != 1 || !tc.isNil(rows[0]) { + t.Fatalf("null %s = %+v, %v, want nil", tc.member, rows, err) + } + if err := json.Unmarshal(mutateFixtureMember(t, netWorth, tc.member, nil), &rows); err == nil { + t.Fatalf("missing %s decoded", tc.member) + } + } + if err := json.Unmarshal(mutateFixtureMember(t, netWorth, "debtDetails.dateIncurred", nil), &entries); err != nil || + len(entries) != 1 || entries[0].DebtDetails == nil || entries[0].DebtDetails.DateIncurred != nil { + t.Fatalf("absent dateIncurred = %+v, %v", entries, err) + } + if encoded, err := json.Marshal(entries[0].DebtDetails); err != nil || string(encoded) != `{}` { + t.Fatalf("re-encoded debtDetails = %s, %v", encoded, err) + } + + const sparse = `[{"cashAndCashEquivalents":121004.5,"mutualFundsAndETFs":34526531.5,` + + `"senateID":"M000355","total":59082540.5,"year":2023}]` + var totals []CongressionalMemberNetWorthAggregate + if err := json.Unmarshal([]byte(sparse), &totals); err != nil || len(totals) != 1 { + t.Fatalf("sparse aggregate = %+v, %v", totals, err) + } + if want := (CongressionalMemberNetWorthAggregate{MemberID: "M000355", Year: 2023, Total: 59_082_540.5, + CashAndCashEquivalents: 121_004.5, MutualFundsAndETFs: 34_526_531.5}); !reflect.DeepEqual(totals[0], want) { + t.Fatalf("sparse aggregate = %+v, want %+v", totals[0], want) + } + encoded, err := json.Marshal(totals) + if err != nil { + t.Fatal(err) + } + assertCanonicalJSON(t, (*jsontext.Value)(&encoded), sparse) + fractionalTotals := strings.Replace(sparse, `"year":2023`, `"year":2023,"realEstate":3000000.5,"stock":8000.5`, 1) + if err := json.Unmarshal([]byte(fractionalTotals), &totals); err != nil || len(totals) != 1 || + !reflect.DeepEqual(totals[0].RealEstate, new(3_000_000.5)) || !reflect.DeepEqual(totals[0].Stock, new(8_000.5)) { + t.Fatalf("fractional aggregate = %+v, %v", totals, err) + } + if err := json.Unmarshal(mutateFixtureMember(t, "congress_senate_net_worth_aggregated.json", "realEstate", jsontext.Value(`null`)), &totals); err != nil || + len(totals) != 1 || totals[0].RealEstate != nil { + t.Fatalf("null realEstate = %+v, %v, want nil", totals, err) + } + for _, member := range []string{"total", "cashAndCashEquivalents", "mutualFundsAndETFs"} { + var rows []CongressionalMemberNetWorthAggregate + if err := json.Unmarshal(mutateFixtureMember(t, "congress_senate_net_worth_aggregated.json", member, nil), &rows); err == nil { + t.Fatalf("missing %s decoded", member) + } + } +} diff --git a/sdk/go/congressional_test.go b/sdk/go/congressional_test.go index 1a7427e..ab46f9b 100644 --- a/sdk/go/congressional_test.go +++ b/sdk/go/congressional_test.go @@ -6,6 +6,7 @@ import ( "fmt" "math" "net/http" + "reflect" "strings" "testing" ) @@ -121,7 +122,7 @@ func TestCongressionalMemberAndNetWorthMethodsUseExactPathsAndWireParameterOrder netWorth, err := client.Congressional.NetWorth(ctx, NewCongressionalNetWorthQuery("P000197")) if err != nil || len(netWorth) != 1 || netWorth[0].DebtDetails == nil || - netWorth[0].DebtDetails.DateIncurred != "September 2007" || netWorth[0].Income != nil { + !reflect.DeepEqual(netWorth[0].DebtDetails.DateIncurred, new("September 2007")) || netWorth[0].Income != nil { t.Fatalf("NetWorth = %+v, %v", netWorth, err) } paged, err := client.Congressional.NetWorth(ctx, NewCongressionalNetWorthQuery("P000197").WithPage(0).WithLimit(250))