diff --git a/crates/fmp-py/python/fmp/_native/congressional/__init__.pyi b/crates/fmp-py/python/fmp/_native/congressional/__init__.pyi index 45e0d12..d64e3fc 100644 --- a/crates/fmp-py/python/fmp/_native/congressional/__init__.pyi +++ b/crates/fmp-py/python/fmp/_native/congressional/__init__.pyi @@ -142,6 +142,13 @@ class CongressionalMemberNetWorthAggregate: typing.Literal["asset_backed_securities"], typing.Literal["business_liabilities"], typing.Literal["mutual_funds_and_etfs"], + typing.Literal["pension_and_retirement_assets"], + typing.Literal["other_amount"], + typing.Literal["other_assets"], + typing.Literal["salary_and_wages"], + typing.Literal["trusts"], + typing.Literal["government_securities"], + typing.Literal["additional_columns"], ] @property @@ -153,7 +160,7 @@ class CongressionalMemberNetWorthAggregate: @property def real_estate_liabilities(self) -> typing.Optional[builtins.float]: ... @property - def cash_and_cash_equivalents(self) -> builtins.float: ... + def cash_and_cash_equivalents(self) -> typing.Optional[builtins.float]: ... @property def business_and_self_employment(self) -> typing.Optional[builtins.float]: ... @property @@ -171,7 +178,21 @@ class CongressionalMemberNetWorthAggregate: @property def business_liabilities(self) -> typing.Optional[builtins.float]: ... @property - def mutual_funds_and_etfs(self) -> builtins.float: ... + def mutual_funds_and_etfs(self) -> typing.Optional[builtins.float]: ... + @property + def pension_and_retirement_assets(self) -> typing.Optional[builtins.float]: ... + @property + def other_amount(self) -> typing.Optional[builtins.float]: ... + @property + def other_assets(self) -> typing.Optional[builtins.float]: ... + @property + def salary_and_wages(self) -> typing.Optional[builtins.float]: ... + @property + def trusts(self) -> typing.Optional[builtins.float]: ... + @property + def government_securities(self) -> typing.Optional[builtins.float]: ... + @property + def additional_columns(self) -> typing.Any: ... def __eq__(self, other: builtins.object, /) -> builtins.bool: ... def __new__( cls, @@ -180,7 +201,7 @@ class CongressionalMemberNetWorthAggregate: year: builtins.int, total: builtins.float, real_estate_liabilities: typing.Optional[builtins.float], - cash_and_cash_equivalents: builtins.float, + cash_and_cash_equivalents: typing.Optional[builtins.float], business_and_self_employment: typing.Optional[builtins.float], real_estate: typing.Optional[builtins.float], ownership_interest: typing.Optional[builtins.float], @@ -189,7 +210,14 @@ class CongressionalMemberNetWorthAggregate: revolving_and_credit_lines: typing.Optional[builtins.float], asset_backed_securities: typing.Optional[builtins.float], business_liabilities: typing.Optional[builtins.float], - mutual_funds_and_etfs: builtins.float, + mutual_funds_and_etfs: typing.Optional[builtins.float], + pension_and_retirement_assets: typing.Optional[builtins.float], + other_amount: typing.Optional[builtins.float], + other_assets: typing.Optional[builtins.float], + salary_and_wages: typing.Optional[builtins.float], + trusts: typing.Optional[builtins.float], + government_securities: typing.Optional[builtins.float], + additional_columns: builtins.str, ) -> CongressionalMemberNetWorthAggregate: ... def __getnewargs_ex__(self) -> tuple[tuple, dict]: ... def __repr__(self) -> builtins.str: ... diff --git a/crates/fmp-py/src/models/congressional.rs b/crates/fmp-py/src/models/congressional.rs index 9e128a2..71d053e 100644 --- a/crates/fmp-py/src/models/congressional.rs +++ b/crates/fmp-py/src/models/congressional.rs @@ -1223,7 +1223,7 @@ pub(crate) struct CongressionalMemberNetWorthAggregate { #[pyo3(get)] pub real_estate_liabilities: Option, #[pyo3(get)] - pub cash_and_cash_equivalents: f64, + pub cash_and_cash_equivalents: Option, #[pyo3(get)] pub business_and_self_employment: Option, #[pyo3(get)] @@ -1241,7 +1241,20 @@ pub(crate) struct CongressionalMemberNetWorthAggregate { #[pyo3(get)] pub business_liabilities: Option, #[pyo3(get)] - pub mutual_funds_and_etfs: f64, + pub mutual_funds_and_etfs: Option, + #[pyo3(get)] + pub pension_and_retirement_assets: Option, + #[pyo3(get)] + pub other_amount: Option, + #[pyo3(get)] + pub other_assets: Option, + #[pyo3(get)] + pub salary_and_wages: Option, + #[pyo3(get)] + pub trusts: Option, + #[pyo3(get)] + pub government_securities: Option, + additional_columns: ::serde_json::Map, } #[gen_stub_pymethods] @@ -1250,13 +1263,13 @@ impl CongressionalMemberNetWorthAggregate { #[new] #[allow(clippy::too_many_arguments)] #[allow(clippy::fn_params_excessive_bools)] - #[pyo3(signature = (*, member_id, year, total, real_estate_liabilities, cash_and_cash_equivalents, business_and_self_employment, real_estate, ownership_interest, stock, options, revolving_and_credit_lines, asset_backed_securities, business_liabilities, mutual_funds_and_etfs))] + #[pyo3(signature = (*, member_id, year, total, real_estate_liabilities, cash_and_cash_equivalents, business_and_self_employment, real_estate, ownership_interest, stock, options, revolving_and_credit_lines, asset_backed_securities, business_liabilities, mutual_funds_and_etfs, pension_and_retirement_assets, other_amount, other_assets, salary_and_wages, trusts, government_securities, additional_columns))] fn new( member_id: String, year: u32, total: f64, real_estate_liabilities: Option, - cash_and_cash_equivalents: f64, + cash_and_cash_equivalents: Option, business_and_self_employment: Option, real_estate: Option, ownership_interest: Option, @@ -1265,9 +1278,24 @@ impl CongressionalMemberNetWorthAggregate { revolving_and_credit_lines: Option, asset_backed_securities: Option, business_liabilities: Option, - mutual_funds_and_etfs: f64, - ) -> Self { - Self { + mutual_funds_and_etfs: Option, + pension_and_retirement_assets: Option, + other_amount: Option, + other_assets: Option, + salary_and_wages: Option, + trusts: Option, + government_securities: Option, + additional_columns: String, + ) -> PyResult { + let additional_columns = ::serde_json::from_str::< + ::serde_json::Map, + >(&additional_columns) + .map_err(|error| { + ::pyo3::exceptions::PyValueError::new_err(format!( + "invalid JSON for field `additional_columns`: {error}" + )) + })?; + Ok(Self { member_id, year, total, @@ -1282,14 +1310,21 @@ impl CongressionalMemberNetWorthAggregate { asset_backed_securities, business_liabilities, mutual_funds_and_etfs, - } + pension_and_retirement_assets, + other_amount, + other_assets, + salary_and_wages, + trusts, + government_securities, + additional_columns, + }) } #[classattr] const __hash__: Option> = None; #[classattr] - #[gen_stub(override_return_type(type_repr = "tuple[typing.Literal['member_id'], typing.Literal['year'], typing.Literal['total'], typing.Literal['real_estate_liabilities'], typing.Literal['cash_and_cash_equivalents'], typing.Literal['business_and_self_employment'], typing.Literal['real_estate'], typing.Literal['ownership_interest'], typing.Literal['stock'], typing.Literal['options'], typing.Literal['revolving_and_credit_lines'], typing.Literal['asset_backed_securities'], typing.Literal['business_liabilities'], typing.Literal['mutual_funds_and_etfs']]", imports = ("typing",)))] + #[gen_stub(override_return_type(type_repr = "tuple[typing.Literal['member_id'], typing.Literal['year'], typing.Literal['total'], typing.Literal['real_estate_liabilities'], typing.Literal['cash_and_cash_equivalents'], typing.Literal['business_and_self_employment'], typing.Literal['real_estate'], typing.Literal['ownership_interest'], typing.Literal['stock'], typing.Literal['options'], typing.Literal['revolving_and_credit_lines'], typing.Literal['asset_backed_securities'], typing.Literal['business_liabilities'], typing.Literal['mutual_funds_and_etfs'], typing.Literal['pension_and_retirement_assets'], typing.Literal['other_amount'], typing.Literal['other_assets'], typing.Literal['salary_and_wages'], typing.Literal['trusts'], typing.Literal['government_securities'], typing.Literal['additional_columns']]", imports = ("typing",)))] fn __match_args__(py: Python<'_>) -> PyResult> { PyTuple::new( py, @@ -1308,6 +1343,13 @@ impl CongressionalMemberNetWorthAggregate { "asset_backed_securities", "business_liabilities", "mutual_funds_and_etfs", + "pension_and_retirement_assets", + "other_amount", + "other_assets", + "salary_and_wages", + "trusts", + "government_securities", + "additional_columns", ], ) } @@ -1365,6 +1407,39 @@ impl CongressionalMemberNetWorthAggregate { "mutual_funds_and_etfs", self.mutual_funds_and_etfs.clone().into_bound_py_any(py)?, )?; + kwargs.set_item( + "pension_and_retirement_assets", + self.pension_and_retirement_assets + .clone() + .into_bound_py_any(py)?, + )?; + kwargs.set_item( + "other_amount", + self.other_amount.clone().into_bound_py_any(py)?, + )?; + kwargs.set_item( + "other_assets", + self.other_assets.clone().into_bound_py_any(py)?, + )?; + kwargs.set_item( + "salary_and_wages", + self.salary_and_wages.clone().into_bound_py_any(py)?, + )?; + kwargs.set_item("trusts", self.trusts.clone().into_bound_py_any(py)?)?; + kwargs.set_item( + "government_securities", + self.government_securities.clone().into_bound_py_any(py)?, + )?; + kwargs.set_item( + "additional_columns", + ::serde_json::to_string(&self.additional_columns) + .map_err(|error| { + ::pyo3::exceptions::PyValueError::new_err(format!( + "failed to serialize field `additional_columns`: {error}" + )) + })? + .into_bound_py_any(py)?, + )?; Ok((PyTuple::empty(py), kwargs)) } #[allow(clippy::clone_on_copy)] @@ -1419,6 +1494,30 @@ impl CongressionalMemberNetWorthAggregate { "mutual_funds_and_etfs", self.mutual_funds_and_etfs.clone().into_bound_py_any(py)?, ), + ( + "pension_and_retirement_assets", + self.pension_and_retirement_assets + .clone() + .into_bound_py_any(py)?, + ), + ( + "other_amount", + self.other_amount.clone().into_bound_py_any(py)?, + ), + ( + "other_assets", + self.other_assets.clone().into_bound_py_any(py)?, + ), + ( + "salary_and_wages", + self.salary_and_wages.clone().into_bound_py_any(py)?, + ), + ("trusts", self.trusts.clone().into_bound_py_any(py)?), + ( + "government_securities", + self.government_securities.clone().into_bound_py_any(py)?, + ), + ("additional_columns", self.additional_columns(py)?), ], ) } @@ -1484,8 +1583,38 @@ impl CongressionalMemberNetWorthAggregate { "mutual_funds_and_etfs", crate::models::convert::DictValue::dict_value(&self.mutual_funds_and_etfs, py)?, )?; + dict.set_item( + "pension_and_retirement_assets", + crate::models::convert::DictValue::dict_value(&self.pension_and_retirement_assets, py)?, + )?; + dict.set_item( + "other_amount", + crate::models::convert::DictValue::dict_value(&self.other_amount, py)?, + )?; + dict.set_item( + "other_assets", + crate::models::convert::DictValue::dict_value(&self.other_assets, py)?, + )?; + dict.set_item( + "salary_and_wages", + crate::models::convert::DictValue::dict_value(&self.salary_and_wages, py)?, + )?; + dict.set_item( + "trusts", + crate::models::convert::DictValue::dict_value(&self.trusts, py)?, + )?; + dict.set_item( + "government_securities", + crate::models::convert::DictValue::dict_value(&self.government_securities, py)?, + )?; + dict.set_item("additional_columns", self.additional_columns(py)?)?; Ok(dict) } + + #[getter] + fn additional_columns<'py>(&self, py: Python<'py>) -> PyResult> { + crate::models::convert::object_to_py(py, &self.additional_columns) + } } impl From @@ -1507,6 +1636,13 @@ impl From None: + """Absent cash or mutual funds decode as ``None``; unmodeled columns land in ``additional_columns``.""" + fixture_server.route("/senate-net-worth-aggregated", load_fixture("congress_senate_net_worth_aggregated.json")) + rows = client.congressional.net_worth_aggregated(MEMBER_ID) + + assert rows[1].cash_and_cash_equivalents is None + assert rows[1].pension_and_retirement_assets == 250_001 + assert rows[2].mutual_funds_and_etfs is None + assert rows[2].government_securities == 15_001 + assert rows[3].salary_and_wages == 174_000 + assert rows[3].additional_columns == {"pensionAndRetirementIncome": 5001, "spousalIncome": 201} + assert rows[4].other_amount == 1_001 + assert rows[4].other_assets == 2_001 + assert rows[4].trusts == 0 + assert isinstance(rows[4].additional_columns, dict) + assert rows[4].additional_columns == {} + assert rows[3].to_dict()["additional_columns"] == {"pensionAndRetirementIncome": 5001, "spousalIncome": 201} + assert pickle.loads(pickle.dumps(rows[3])) == rows[3] def test_net_worth_aggregated_with_totals_col(client: Any, fixture_server: FixtureServer) -> None: @@ -186,7 +210,7 @@ def test_net_worth_aggregated_with_totals_col(client: Any, fixture_server: Fixtu rows = client.congressional.net_worth_aggregated(MEMBER_ID, totals_col="stock") assert fixture_server.requests[0].target == "/senate-net-worth-aggregated?senateID=P000197&totalsCol=stock" - assert len(rows) == 1 + assert len(rows) == 5 assert rows[0].total == 225_219_551 diff --git a/crates/fmp-py/tests/typecheck_contract.py b/crates/fmp-py/tests/typecheck_contract.py index cae75ef..16ac9fd 100644 --- a/crates/fmp-py/tests/typecheck_contract.py +++ b/crates/fmp-py/tests/typecheck_contract.py @@ -209,7 +209,9 @@ def check_congressional_contract(client: FmpClient) -> None: value: float | None = entries[0].value total: float = totals[0].total options: float | None = totals[0].options - _ = (disclosed, gains, born, years_active, value_range, value, total, options) + cash: float | None = totals[0].cash_and_cash_equivalents + other: float | None = totals[0].other_amount + _ = (disclosed, gains, born, years_active, value_range, value, total, options, cash, other) def check_funds_contract(client: FmpClient) -> None: diff --git a/crates/libfmp/src/responses/congressional.rs b/crates/libfmp/src/responses/congressional.rs index f2f83b1..ecbca98 100644 --- a/crates/libfmp/src/responses/congressional.rs +++ b/crates/libfmp/src/responses/congressional.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Deserializer, Serialize}; use crate::{ - codecs::{DynamicJson, OpaqueDateText, TitleCaseBoolFlag}, + codecs::{DynamicJson, DynamicObject, OpaqueDateText, TitleCaseBoolFlag}, types::{CalendarYear, CongressionalMemberId, Date, FormType, MarketValue}, }; @@ -145,8 +145,10 @@ pub struct CongressionalMemberNetWorth { /// Aggregated congressional net-worth totals for one filing year. /// -/// Only the member, year, total, cash and mutual-fund members are sent on -/// every row; the other amounts are `None` when the key is absent. +/// Only the member, year and total are sent on every row; every amount is +/// `None` when its key is absent. Columns without a typed member, such as +/// `pensionAndRetirementIncome` or `spousalIncome`, are kept in +/// `additional_columns` instead of being dropped. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[non_exhaustive] #[serde(rename_all = "camelCase")] @@ -159,8 +161,9 @@ pub struct CongressionalMemberNetWorthAggregate { #[serde(skip_serializing_if = "Option::is_none")] #[serde(serialize_with = "crate::codecs::integral_f64::serialize_option")] pub real_estate_liabilities: Option, - #[serde(serialize_with = "crate::codecs::integral_f64::serialize")] - pub cash_and_cash_equivalents: MarketValue, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(serialize_with = "crate::codecs::integral_f64::serialize_option")] + pub cash_and_cash_equivalents: Option, #[serde(skip_serializing_if = "Option::is_none")] #[serde(serialize_with = "crate::codecs::integral_f64::serialize_option")] pub business_and_self_employment: Option, @@ -186,6 +189,29 @@ pub struct CongressionalMemberNetWorthAggregate { #[serde(serialize_with = "crate::codecs::integral_f64::serialize_option")] pub business_liabilities: Option, #[serde(rename = "mutualFundsAndETFs")] - #[serde(serialize_with = "crate::codecs::integral_f64::serialize")] - pub mutual_funds_and_etfs: MarketValue, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(serialize_with = "crate::codecs::integral_f64::serialize_option")] + pub mutual_funds_and_etfs: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(serialize_with = "crate::codecs::integral_f64::serialize_option")] + pub pension_and_retirement_assets: Option, + #[serde(rename = "Other")] + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(serialize_with = "crate::codecs::integral_f64::serialize_option")] + pub other_amount: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(serialize_with = "crate::codecs::integral_f64::serialize_option")] + pub other_assets: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(serialize_with = "crate::codecs::integral_f64::serialize_option")] + pub salary_and_wages: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(serialize_with = "crate::codecs::integral_f64::serialize_option")] + pub trusts: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(serialize_with = "crate::codecs::integral_f64::serialize_option")] + pub government_securities: Option, + /// Every column without a typed member above, keyed by its wire name. + #[serde(flatten)] + pub additional_columns: DynamicObject, } diff --git a/crates/libfmp/tests/congressional_net_worth_endpoints.rs b/crates/libfmp/tests/congressional_net_worth_endpoints.rs index ea093ac..e0e05d9 100644 --- a/crates/libfmp/tests/congressional_net_worth_endpoints.rs +++ b/crates/libfmp/tests/congressional_net_worth_endpoints.rs @@ -13,7 +13,7 @@ use libfmp::{ }, metadata::{AccessRequirement, EndpointBounds, GeographicAvailability}, }, - error::ErrorCategory, + error::{DecodeErrorKind, ErrorCategory}, responses::congressional::{CongressionalMemberNetWorth, CongressionalMemberNetWorthAggregate}, transport::HttpMethod, types::{CongressionalMemberId, Limit, Page}, @@ -291,15 +291,11 @@ fn itemized_fractional_value_and_null_members_decode_exactly() { assert_eq!(entry.value_range.unwrap().max, None); } -const AGGREGATE_REQUIRED: [&str; 5] = [ - "senateID", - "year", - "total", +const AGGREGATE_REQUIRED: [&str; 3] = ["senateID", "year", "total"]; + +const AGGREGATE_OMITTABLE: [&str; 17] = [ "cashAndCashEquivalents", "mutualFundsAndETFs", -]; - -const AGGREGATE_OMITTABLE: [&str; 9] = [ "realEstate", "stock", "realEstateLiabilities", @@ -309,6 +305,12 @@ const AGGREGATE_OMITTABLE: [&str; 9] = [ "revolvingAndCreditLines", "assetBackedSecurities", "businessLiabilities", + "pensionAndRetirementAssets", + "Other", + "otherAssets", + "salaryAndWages", + "trusts", + "governmentSecurities", ]; #[test] @@ -355,13 +357,12 @@ fn aggregated_required_fields_are_strict_and_omittable_fields_may_be_absent() { sparse.as_object_mut().unwrap().remove(field); } sparse["total"] = serde_json::json!(59082540.5); - sparse["cashAndCashEquivalents"] = serde_json::json!(121004.5); - sparse["mutualFundsAndETFs"] = serde_json::json!(34526531.5); let totals = serde_json::from_value::(sparse.clone()).unwrap(); assert_eq!(totals.total, 59_082_540.5); - assert_eq!(totals.cash_and_cash_equivalents, 121_004.5); - assert_eq!(totals.mutual_funds_and_etfs, 34_526_531.5); + assert_eq!(totals.cash_and_cash_equivalents, None); + assert_eq!(totals.mutual_funds_and_etfs, None); + assert!(totals.additional_columns.is_empty()); assert_eq!(totals.real_estate, None); assert_eq!(totals.stock, None); assert_eq!(totals.real_estate_liabilities, None); @@ -386,6 +387,80 @@ fn aggregated_required_fields_are_strict_and_omittable_fields_may_be_absent() { assert_eq!(totals.stock, Some(8_000.5)); } +#[test] +fn aggregated_rows_keep_typed_and_additional_columns_and_round_trip() { + let rows = + serde_json::from_slice::>(AGGREGATED).unwrap(); + assert_eq!(rows.len(), 5); + assert!(rows[0].additional_columns.is_empty()); + assert_eq!(rows[1].cash_and_cash_equivalents, None); + assert_eq!(rows[1].pension_and_retirement_assets, Some(250_001.0)); + assert_eq!(rows[2].mutual_funds_and_etfs, None); + assert_eq!(rows[2].government_securities, Some(15_001.0)); + assert_eq!(rows[3].salary_and_wages, Some(174_000.0)); + assert_eq!( + serde_json::Value::Object(rows[3].additional_columns.clone()), + serde_json::json!({"pensionAndRetirementIncome": 5001, "spousalIncome": 201}) + ); + assert_eq!(rows[4].other_amount, Some(1_001.0)); + assert_eq!(rows[4].other_assets, Some(2_001.0)); + assert_eq!(rows[4].trusts, Some(0.0)); + assert!(rows[4].additional_columns.is_empty()); + + for row in &rows { + let encoded = serde_json::to_string(row).unwrap(); + let members = serde_json::to_value(row).unwrap(); + let members = members.as_object().unwrap(); + assert_eq!(encoded.matches(':').count(), members.len(), "{encoded}"); + for key in members.keys() { + assert_eq!( + encoded.matches(&format!("\"{key}\":")).count(), + 1, + "{encoded}" + ); + } + } + assert_eq!( + serde_json::to_value(&rows).unwrap(), + serde_json::from_slice::(AGGREGATED).unwrap() + ); +} + +#[tokio::test] +async fn aggregated_type_and_missing_member_errors_keep_their_row_path() { + let row = serde_json::from_slice::(AGGREGATED).unwrap()[3].clone(); + let mut wrong_type = row.clone(); + wrong_type["total"] = serde_json::json!("not a number"); + let mut missing = row; + missing.as_object_mut().unwrap().remove("total"); + let body = + |row: serde_json::Value| -> &'static [u8] { serde_json::to_vec(&[row]).unwrap().leak() }; + let executor = Arc::new(FixtureExecutor::new([ + json_fixture(body(wrong_type)), + json_fixture(body(missing)), + ])); + let client = Client::builder() + .authentication(Authentication::fmp_header("secret")) + .executor(executor) + .build() + .unwrap(); + let member_id = || CongressionalMemberId::new("R000619").unwrap(); + + let error = client + .congressional_net_worth_aggregated(member_id()) + .await + .unwrap_err(); + assert_eq!(error.category(), ErrorCategory::Decode); + assert_eq!(error.decode_path(), Some("[0].total")); + assert_eq!(error.decode_kind(), Some(DecodeErrorKind::WrongType)); + let error = client + .congressional_net_worth_aggregated(member_id()) + .await + .unwrap_err(); + assert_eq!(error.decode_path(), Some("[0].total")); + assert_eq!(error.decode_kind(), Some(DecodeErrorKind::MissingMember)); +} + #[tokio::test] async fn direct_fmp_header_and_query_auth_apply_to_both_routes() { for (authentication, query_auth, expected_header) in [ diff --git a/crates/libfmp/tests/fixtures/congress_senate_net_worth_aggregated.json b/crates/libfmp/tests/fixtures/congress_senate_net_worth_aggregated.json index ef043ea..491ab5c 100644 --- a/crates/libfmp/tests/fixtures/congress_senate_net_worth_aggregated.json +++ b/crates/libfmp/tests/fixtures/congress_senate_net_worth_aggregated.json @@ -14,5 +14,39 @@ "stock": 136748525, "total": 225219551, "year": 2024 + }, + { + "mutualFundsAndETFs": 15001, + "pensionAndRetirementAssets": 250001, + "senateID": "C001123", + "total": 265002, + "year": 2023 + }, + { + "cashAndCashEquivalents": 50001, + "governmentSecurities": 15001, + "senateID": "R000619", + "total": 65002, + "year": 2022 + }, + { + "cashAndCashEquivalents": 100001, + "mutualFundsAndETFs": 1001, + "pensionAndRetirementIncome": 5001, + "salaryAndWages": 174000, + "senateID": "R000619", + "spousalIncome": 201, + "total": 280204, + "year": 2021 + }, + { + "Other": 1001, + "cashAndCashEquivalents": 15001, + "mutualFundsAndETFs": 50001, + "otherAssets": 2001, + "senateID": "R000619", + "total": 68004, + "trusts": 0, + "year": 2020 } ] diff --git a/docs/adr/0033-nullable-response-members.md b/docs/adr/0033-nullable-response-members.md index cb9a3ae..c817385 100644 --- a/docs/adr/0033-nullable-response-members.md +++ b/docs/adr/0033-nullable-response-members.md @@ -68,6 +68,19 @@ evidence is now a second source, so both need a rule. `senate-net-worth` rows. The same audit saw fractional amounts (`32500.5`) on members typed as integers; those became `MarketValue` (`f64`) under the next rule. + A 92-member survey for + [#407](https://github.com/bitbrew-dev/libfmp/issues/407) (538 rows) then + saw `cashAndCashEquivalents` (24 rows) and `mutualFundsAndETFs` (2 rows) + absent too, so both became optional-when-absent. +- A response whose column set is open-ended keeps the columns it does not + type in a rest map instead of dropping them. In a derived model the + sanctioned shape is `#[serde(flatten)]` on a bare `DynamicObject`; a + hand-written `Deserialize` stays reserved for header-only models such as + `FinancialReportJson`. First case, #407: + `CongressionalMemberNetWorthAggregate` types the six most frequent extra + columns (`pensionAndRetirementAssets`, `Other` as `other_amount`, + `otherAssets`, `salaryAndWages`, `trusts`, `governmentSecurities`) and + keeps the rest in `additional_columns`. - A member whose non-null value has the wrong type (a year where a date is documented) is a separate type fix, not an `Option`. - Every change is breaking and ships in a minor release with a @@ -88,6 +101,9 @@ anything else unmarshals into the struct with failures reported through A plain `Option` without a codec is `Codec::Plain`: the key may be absent or null, and `skip_serializing_if = "Option::is_none"` adds `omitzero` so a re-encoded row omits it as serde does. +A `#[serde(flatten)]` bare `DynamicObject` maps to `Codec::Embedded`, a +`jsontext.Value` with the `embed` option, so json/v2 keeps every member no +named field claims and re-encodes it. ## Alternatives considered diff --git a/sdk/go/congressional_models.go b/sdk/go/congressional_models.go index 55478a5..e7b5287 100644 --- a/sdk/go/congressional_models.go +++ b/sdk/go/congressional_models.go @@ -534,45 +534,60 @@ func (m *CongressionalMemberNetWorth) UnmarshalJSONFrom(dec *jsontext.Decoder) e // CongressionalMemberNetWorthAggregate is aggregated congressional net-worth // totals for one filing year. type CongressionalMemberNetWorthAggregate struct { - MemberID string `json:"senateID"` - Year uint32 `json:"year"` - Total float64 `json:"total"` - RealEstateLiabilities *float64 `json:"realEstateLiabilities,omitzero"` - CashAndCashEquivalents float64 `json:"cashAndCashEquivalents"` - BusinessAndSelfEmployment *float64 `json:"businessAndSelfEmployment,omitzero"` - RealEstate *float64 `json:"realEstate,omitzero"` - OwnershipInterest *float64 `json:"ownershipInterest,omitzero"` - Stock *float64 `json:"stock,omitzero"` - Options *float64 `json:"options,omitzero"` - RevolvingAndCreditLines *float64 `json:"revolvingAndCreditLines,omitzero"` - AssetBackedSecurities *float64 `json:"assetBackedSecurities,omitzero"` - BusinessLiabilities *float64 `json:"businessLiabilities,omitzero"` - MutualFundsAndETFs float64 `json:"mutualFundsAndETFs"` + MemberID string `json:"senateID"` + Year uint32 `json:"year"` + Total float64 `json:"total"` + RealEstateLiabilities *float64 `json:"realEstateLiabilities,omitzero"` + CashAndCashEquivalents *float64 `json:"cashAndCashEquivalents,omitzero"` + BusinessAndSelfEmployment *float64 `json:"businessAndSelfEmployment,omitzero"` + RealEstate *float64 `json:"realEstate,omitzero"` + OwnershipInterest *float64 `json:"ownershipInterest,omitzero"` + Stock *float64 `json:"stock,omitzero"` + Options *float64 `json:"options,omitzero"` + RevolvingAndCreditLines *float64 `json:"revolvingAndCreditLines,omitzero"` + AssetBackedSecurities *float64 `json:"assetBackedSecurities,omitzero"` + BusinessLiabilities *float64 `json:"businessLiabilities,omitzero"` + MutualFundsAndETFs *float64 `json:"mutualFundsAndETFs,omitzero"` + PensionAndRetirementAssets *float64 `json:"pensionAndRetirementAssets,omitzero"` + OtherAmount *float64 `json:"Other,omitzero"` + OtherAssets *float64 `json:"otherAssets,omitzero"` + SalaryAndWages *float64 `json:"salaryAndWages,omitzero"` + Trusts *float64 `json:"trusts,omitzero"` + GovernmentSecurities *float64 `json:"governmentSecurities,omitzero"` + AdditionalColumns jsontext.Value `json:",embed"` } // congressionalMemberNetWorthAggregateShadow mirrors // CongressionalMemberNetWorthAggregate with a pointer or raw value for every // required member so a missing or null member is observable after decoding. type congressionalMemberNetWorthAggregateShadow struct { - MemberID *string `json:"senateID"` - Year *uint32 `json:"year"` - Total *float64 `json:"total"` - RealEstateLiabilities *float64 `json:"realEstateLiabilities"` - CashAndCashEquivalents *float64 `json:"cashAndCashEquivalents"` - BusinessAndSelfEmployment *float64 `json:"businessAndSelfEmployment"` - RealEstate *float64 `json:"realEstate"` - OwnershipInterest *float64 `json:"ownershipInterest"` - Stock *float64 `json:"stock"` - Options *float64 `json:"options"` - RevolvingAndCreditLines *float64 `json:"revolvingAndCreditLines"` - AssetBackedSecurities *float64 `json:"assetBackedSecurities"` - BusinessLiabilities *float64 `json:"businessLiabilities"` - MutualFundsAndETFs *float64 `json:"mutualFundsAndETFs"` + MemberID *string `json:"senateID"` + Year *uint32 `json:"year"` + Total *float64 `json:"total"` + RealEstateLiabilities *float64 `json:"realEstateLiabilities"` + CashAndCashEquivalents *float64 `json:"cashAndCashEquivalents"` + BusinessAndSelfEmployment *float64 `json:"businessAndSelfEmployment"` + RealEstate *float64 `json:"realEstate"` + OwnershipInterest *float64 `json:"ownershipInterest"` + Stock *float64 `json:"stock"` + Options *float64 `json:"options"` + RevolvingAndCreditLines *float64 `json:"revolvingAndCreditLines"` + AssetBackedSecurities *float64 `json:"assetBackedSecurities"` + BusinessLiabilities *float64 `json:"businessLiabilities"` + MutualFundsAndETFs *float64 `json:"mutualFundsAndETFs"` + PensionAndRetirementAssets *float64 `json:"pensionAndRetirementAssets"` + OtherAmount *float64 `json:"Other"` + OtherAssets *float64 `json:"otherAssets"` + SalaryAndWages *float64 `json:"salaryAndWages"` + Trusts *float64 `json:"trusts"` + GovernmentSecurities *float64 `json:"governmentSecurities"` + AdditionalColumns jsontext.Value `json:",embed"` } // UnmarshalJSONFrom decodes one JSON object and rejects it with a Decode // error naming the first required member that is missing or null, as the -// Rust decoder does. Unknown members are ignored. +// Rust decoder does. Every member no named field claims is kept in +// AdditionalColumns. func (m *CongressionalMemberNetWorthAggregate) UnmarshalJSONFrom(dec *jsontext.Decoder) error { var shadow congressionalMemberNetWorthAggregateShadow if err := json.UnmarshalDecode(dec, &shadow); err != nil { @@ -585,26 +600,32 @@ func (m *CongressionalMemberNetWorthAggregate) UnmarshalJSONFrom(dec *jsontext.D return missingMemberError("CongressionalMemberNetWorthAggregate", "year") case shadow.Total == nil: return missingMemberError("CongressionalMemberNetWorthAggregate", "total") - case shadow.CashAndCashEquivalents == nil: - return missingMemberError("CongressionalMemberNetWorthAggregate", "cashAndCashEquivalents") - case shadow.MutualFundsAndETFs == nil: - return missingMemberError("CongressionalMemberNetWorthAggregate", "mutualFundsAndETFs") + } + if len(shadow.AdditionalColumns) == 0 { + shadow.AdditionalColumns = jsontext.Value("{}") } *m = CongressionalMemberNetWorthAggregate{ - MemberID: *shadow.MemberID, - Year: *shadow.Year, - Total: *shadow.Total, - RealEstateLiabilities: shadow.RealEstateLiabilities, - CashAndCashEquivalents: *shadow.CashAndCashEquivalents, - BusinessAndSelfEmployment: shadow.BusinessAndSelfEmployment, - RealEstate: shadow.RealEstate, - OwnershipInterest: shadow.OwnershipInterest, - Stock: shadow.Stock, - Options: shadow.Options, - RevolvingAndCreditLines: shadow.RevolvingAndCreditLines, - AssetBackedSecurities: shadow.AssetBackedSecurities, - BusinessLiabilities: shadow.BusinessLiabilities, - MutualFundsAndETFs: *shadow.MutualFundsAndETFs, + MemberID: *shadow.MemberID, + Year: *shadow.Year, + Total: *shadow.Total, + RealEstateLiabilities: shadow.RealEstateLiabilities, + CashAndCashEquivalents: shadow.CashAndCashEquivalents, + BusinessAndSelfEmployment: shadow.BusinessAndSelfEmployment, + RealEstate: shadow.RealEstate, + OwnershipInterest: shadow.OwnershipInterest, + Stock: shadow.Stock, + Options: shadow.Options, + RevolvingAndCreditLines: shadow.RevolvingAndCreditLines, + AssetBackedSecurities: shadow.AssetBackedSecurities, + BusinessLiabilities: shadow.BusinessLiabilities, + MutualFundsAndETFs: shadow.MutualFundsAndETFs, + PensionAndRetirementAssets: shadow.PensionAndRetirementAssets, + OtherAmount: shadow.OtherAmount, + OtherAssets: shadow.OtherAssets, + SalaryAndWages: shadow.SalaryAndWages, + Trusts: shadow.Trusts, + GovernmentSecurities: shadow.GovernmentSecurities, + AdditionalColumns: shadow.AdditionalColumns, } return nil } diff --git a/sdk/go/congressional_parity_test.go b/sdk/go/congressional_parity_test.go index 8401685..9f1f87f 100644 --- a/sdk/go/congressional_parity_test.go +++ b/sdk/go/congressional_parity_test.go @@ -142,12 +142,26 @@ func TestDocumentedMemberAndNetWorthFixturesDecodeExactValues(t *testing.T) { totals := assertFixtureParity[CongressionalMemberNetWorthAggregate](t, "congress_senate_net_worth_aggregated.json") if want := (CongressionalMemberNetWorthAggregate{MemberID: "P000197", Year: 2024, Total: 225_219_551, - RealEstateLiabilities: new(27_000_005.0), CashAndCashEquivalents: 291_009, BusinessAndSelfEmployment: new(0.0), + RealEstateLiabilities: new(27_000_005.0), CashAndCashEquivalents: new(291_009.0), BusinessAndSelfEmployment: new(0.0), RealEstate: new(45_032_504.0), OwnershipInterest: new(70_140_014.0), Stock: new(136_748_525.0), Options: new(0.0), RevolvingAndCreditLines: new(1_500_002.0), AssetBackedSecurities: new(4_475_006.0), BusinessLiabilities: new(3_000_001.0), - MutualFundsAndETFs: 32_501}); len(totals) != 1 || !reflect.DeepEqual(totals[0], want) { + MutualFundsAndETFs: new(32_501.0), AdditionalColumns: jsontext.Value("{}")}); len(totals) != 5 || !reflect.DeepEqual(totals[0], want) { t.Fatalf("congress_senate_net_worth_aggregated = %+v", totals) } + if want := (CongressionalMemberNetWorthAggregate{MemberID: "R000619", Year: 2021, Total: 280_204, + CashAndCashEquivalents: new(100_001.0), MutualFundsAndETFs: new(1_001.0), SalaryAndWages: new(174_000.0), + AdditionalColumns: jsontext.Value(`{"pensionAndRetirementIncome":5001,"spousalIncome":201}`)}); !reflect.DeepEqual(totals[3], want) { + t.Fatalf("aggregate with additional columns = %+v, want %+v", totals[3], want) + } + if want := (CongressionalMemberNetWorthAggregate{MemberID: "R000619", Year: 2020, Total: 68_004, + CashAndCashEquivalents: new(15_001.0), MutualFundsAndETFs: new(50_001.0), OtherAmount: new(1_001.0), + OtherAssets: new(2_001.0), Trusts: new(0.0), AdditionalColumns: jsontext.Value("{}")}); !reflect.DeepEqual(totals[4], want) { + t.Fatalf("aggregate with Other = %+v, want %+v", totals[4], want) + } + if totals[1].CashAndCashEquivalents != nil || !reflect.DeepEqual(totals[1].PensionAndRetirementAssets, new(250_001.0)) || + totals[2].MutualFundsAndETFs != nil || !reflect.DeepEqual(totals[2].GovernmentSecurities, new(15_001.0)) { + t.Fatalf("aggregate absent cash or mutual funds = %+v, %+v", totals[1], totals[2]) + } if encoded, err := json.Marshal(totals[0]); err != nil || !strings.Contains(string(encoded), `"mutualFundsAndETFs":32501`) { t.Fatalf("re-encoded aggregate = %s, %v", encoded, err) } @@ -299,14 +313,13 @@ func TestCongressionalNetWorthFractionalNullAndAbsentMembers(t *testing.T) { t.Fatalf("re-encoded debtDetails = %s, %v", encoded, err) } - const sparse = `[{"cashAndCashEquivalents":121004.5,"mutualFundsAndETFs":34526531.5,` + - `"senateID":"M000355","total":59082540.5,"year":2023}]` + const sparse = `[{"senateID":"M000355","total":59082540.5,"year":2023}]` var totals []CongressionalMemberNetWorthAggregate if err := json.Unmarshal([]byte(sparse), &totals); err != nil || len(totals) != 1 { t.Fatalf("sparse aggregate = %+v, %v", totals, err) } if want := (CongressionalMemberNetWorthAggregate{MemberID: "M000355", Year: 2023, Total: 59_082_540.5, - CashAndCashEquivalents: 121_004.5, MutualFundsAndETFs: 34_526_531.5}); !reflect.DeepEqual(totals[0], want) { + AdditionalColumns: jsontext.Value("{}")}); !reflect.DeepEqual(totals[0], want) { t.Fatalf("sparse aggregate = %+v, want %+v", totals[0], want) } encoded, err := json.Marshal(totals) @@ -323,11 +336,8 @@ func TestCongressionalNetWorthFractionalNullAndAbsentMembers(t *testing.T) { len(totals) != 1 || totals[0].RealEstate != nil { t.Fatalf("null realEstate = %+v, %v, want nil", totals, err) } - for _, member := range []string{"total", "cashAndCashEquivalents", "mutualFundsAndETFs"} { - var rows []CongressionalMemberNetWorthAggregate - if err := json.Unmarshal(mutateFixtureMember(t, "congress_senate_net_worth_aggregated.json", member, nil), &rows); err == nil { - t.Fatalf("missing %s decoded", member) - } + if err := json.Unmarshal(mutateFixtureMember(t, "congress_senate_net_worth_aggregated.json", "total", nil), &totals); err == nil { + t.Fatal("missing total decoded") } } diff --git a/sdk/go/congressional_test.go b/sdk/go/congressional_test.go index ab46f9b..3982ea7 100644 --- a/sdk/go/congressional_test.go +++ b/sdk/go/congressional_test.go @@ -130,12 +130,12 @@ func TestCongressionalMemberAndNetWorthMethodsUseExactPathsAndWireParameterOrder t.Fatalf("NetWorth paged = %+v, %v", paged, err) } totals, err := client.Congressional.NetWorthAggregated(ctx, NewCongressionalNetWorthAggregatedQuery("P000197")) - if err != nil || len(totals) != 1 || totals[0].Total != 225_219_551 { + if err != nil || len(totals) != 5 || totals[0].Total != 225_219_551 { t.Fatalf("NetWorthAggregated = %+v, %v", totals, err) } columns, err := client.Congressional.NetWorthAggregated(ctx, NewCongressionalNetWorthAggregatedQuery("P000197").WithTotalsCol("")) - if err != nil || len(columns) != 1 { + if err != nil || len(columns) != 5 { t.Fatalf("NetWorthAggregated totalsCol = %+v, %v", columns, err) } assertCongressionalRequests(t, rec, 8)