Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions crates/fmp-py-gen/src/bin/gen_go/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,7 @@ fn render_model(model: &ModelPlan, out: &mut String) {
| Codec::RequiredNumber
| Codec::Count
| Codec::EmptyOrNullString
| Codec::NullTextString
| Codec::EmptyDate
| Codec::EmptyOrNullDate => local_name(rust_name),
};
Expand Down Expand Up @@ -314,6 +315,7 @@ fn render_required_switch(model: &ModelPlan, out: &mut String) {
| Codec::RequiredNumber
| Codec::Count
| Codec::EmptyOrNullString
| Codec::NullTextString
| Codec::EmptyDate
| Codec::EmptyOrNullDate
| Codec::DynamicJson => format!("len(shadow.{}) == 0", field.name),
Expand Down Expand Up @@ -356,13 +358,18 @@ fn render_codec_block(model: &str, field: &GoField, local: &str, out: &mut Strin
field.public_ty, field.name, field.name
);
}
Codec::EmptyOrNullString => {
Codec::EmptyOrNullString | Codec::NullTextString => {
let inner = field.public_ty.trim_start_matches('*');
let sentinel = if field.codec == Codec::NullTextString {
"NULL"
} else {
""
};
let _ = writeln!(
out,
"\tvar {local} {}\n\tif shadow.{}.Kind() != 'n' {{\n\t\tvar value {inner}\n\
\t\tif err := json.Unmarshal(shadow.{}, &value); err != nil {{\n\
\t\t\treturn err\n\t\t}}\n\t\tif value != \"\" {{\n\t\t\t{local} = &value\n\t\t}}\n\t}}",
\t\t\treturn err\n\t\t}}\n\t\tif value != {sentinel:?} {{\n\t\t\t{local} = &value\n\t\t}}\n\t}}",
field.public_ty, field.name, field.name
);
}
Expand Down
48 changes: 48 additions & 0 deletions crates/fmp-py-gen/src/bin/gen_go/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -697,6 +697,54 @@ fn empty_or_null_codec_decodes_empty_and_null_to_nil_through_the_raw_shadow() {
}
}

#[test]
fn null_text_codec_decodes_the_null_text_and_null_to_nil_through_the_raw_shadow() {
let aliases = BTreeMap::new();
let structs = Vec::new();
let table = TypeTable::new(&structs, &aliases);
let codec = "crate::codecs::null_text::deserialize";
let def = row(vec![field(
"entity_org_type",
"Option<NumericString>",
deserialize_with(codec),
)]);
let mapped = table.go_field(&def, &def.fields[0]).expect("maps");
assert_eq!(
(
mapped.public_ty.as_str(),
mapped.shadow_ty.as_str(),
mapped.codec,
mapped.required_key(),
),
("*string", "jsontext.Value", Codec::NullTextString, true)
);
let models = plan_models("test", &[&def], &table).expect("plans");
let rendered = render_models("test", &models);
for expected in [
"EntityOrgType *string `json:\"entityOrgType\"`",
"EntityOrgType jsontext.Value `json:\"entityOrgType\"`",
"case len(shadow.EntityOrgType) == 0:",
"var entityOrgType *string\n\tif shadow.EntityOrgType.Kind() != 'n' {",
"if value != \"NULL\" {\n\t\t\tentityOrgType = &value\n\t\t}",
"EntityOrgType: entityOrgType,",
] {
assert!(
rendered.contains(expected),
"missing {expected:?} in\n{rendered}"
);
}

for (name, ty) in [
("bare", "NumericString"),
("price", "Option<f64>"),
("many", "Option<Vec<NumericString>>"),
] {
let def = row(vec![field(name, ty, deserialize_with(codec))]);
let error = table.go_field(&def, &def.fields[0]).expect_err(name);
assert!(error.contains("null_text"), "{name}: {error}");
}
}

#[test]
fn skip_serializing_if_none_marks_only_the_public_tag_omitzero() {
let aliases = BTreeMap::new();
Expand Down
17 changes: 17 additions & 0 deletions crates/fmp-py-gen/src/bin/gen_go/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,11 @@ pub(crate) enum Codec {
/// `Option` of a string-backed type: the key is required, and `""` and
/// null are both absent (ADR 0033). The shadow keeps the raw value.
EmptyOrNullString,
/// `deserialize_with = "crate::codecs::null_text::deserialize"` on an
/// `Option` of a string-backed type: the key is required, and the text
/// `"NULL"` and null are both absent (ADR 0033). The shadow keeps the raw
/// value.
NullTextString,
/// A `DynamicObject` that holds every member the named fields do not
/// claim, as serde `flatten` on a map does: `#[serde(flatten)]`, or the
/// single `DynamicObject` field of a struct with a hand-written
Expand All @@ -54,6 +59,9 @@ const COUNT_CODEC: &str = "crate::codecs::count::deserialize";
/// The `deserialize_with` path of the Rust empty-or-null typed-code codec.
const EMPTY_OR_NULL_CODEC: &str = "crate::codecs::empty_or_null::deserialize";

/// The `deserialize_with` path of the Rust `"NULL"`-text codec.
const NULL_TEXT_CODEC: &str = "crate::codecs::null_text::deserialize";

/// One Go struct member derived from a Rust field.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct GoField {
Expand Down Expand Up @@ -88,6 +96,7 @@ impl GoField {
| Codec::RequiredNumber
| Codec::Count
| Codec::EmptyOrNullString
| Codec::NullTextString
| Codec::EmptyDate
| Codec::EmptyOrNullDate
| Codec::DynamicJson
Expand Down Expand Up @@ -207,6 +216,14 @@ impl<'a> TypeTable<'a> {
"the empty_or_null codec needs an Option of a string-backed type".to_string(),
));
}
(Some(NULL_TEXT_CODEC), BaseKind::Scalar, [Wrap::Option]) if base.go == "string" => {
(Codec::NullTextString, "jsontext.Value".to_string())
}
(Some(NULL_TEXT_CODEC), _, _) => {
return Err(fail(
"the null_text codec needs an Option of a string-backed type".to_string(),
));
}
(Some("required_option"), _, _) => {
return Err(fail(
"required_option needs an Option<scalar, Number, or DynamicJson> field"
Expand Down
4 changes: 2 additions & 2 deletions crates/fmp-py/python/fmp/_native/funds/__init__.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -480,7 +480,7 @@ class FundDisclosureSearchResult:
@property
def entity_name(self) -> builtins.str: ...
@property
def entity_org_type(self) -> builtins.str: ...
def entity_org_type(self) -> typing.Optional[builtins.str]: ...
@property
def series_name(self) -> builtins.str: ...
@property
Expand All @@ -504,7 +504,7 @@ class FundDisclosureSearchResult:
class_id: builtins.str,
series_id: builtins.str,
entity_name: builtins.str,
entity_org_type: builtins.str,
entity_org_type: typing.Optional[builtins.str],
series_name: builtins.str,
class_name: builtins.str,
reporting_file_number: builtins.str,
Expand Down
6 changes: 3 additions & 3 deletions crates/fmp-py/src/models/funds.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1616,7 +1616,7 @@ pub(crate) struct FundDisclosureSearchResult {
#[pyo3(get)]
pub entity_name: String,
#[pyo3(get)]
pub entity_org_type: String,
pub entity_org_type: Option<String>,
#[pyo3(get)]
pub series_name: String,
#[pyo3(get)]
Expand Down Expand Up @@ -1646,7 +1646,7 @@ impl FundDisclosureSearchResult {
class_id: String,
series_id: String,
entity_name: String,
entity_org_type: String,
entity_org_type: Option<String>,
series_name: String,
class_name: String,
reporting_file_number: String,
Expand Down Expand Up @@ -1834,7 +1834,7 @@ impl From<libfmp::responses::funds::FundDisclosureSearchResult> for FundDisclosu
class_id: value.class_id,
series_id: value.series_id,
entity_name: value.entity_name,
entity_org_type: value.entity_org_type.into_inner(),
entity_org_type: value.entity_org_type.map(|value| value.into_inner()),
series_name: value.series_name,
class_name: value.class_name,
reporting_file_number: value.reporting_file_number,
Expand Down
18 changes: 18 additions & 0 deletions crates/fmp-py/tests/test_funds_b.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,24 @@ def test_disclosure_rows_decode_omitted_members_as_none(client: Any, fixture_ser
assert client.funds.search_disclosure_holders(FEDERATED)[0].address is None


def test_search_entity_org_type_null_text_decodes_as_none(
client: Any, fixture_server: FixtureServer, errors: SimpleNamespace
) -> None:
"""The ``"NULL"`` text in ``entityOrgType`` is ``None``; any other non-numeric text is a decode error."""
search = load_fixture("fund_disclosure_holder_search.json")
search[0]["entityOrgType"] = "NULL"
fixture_server.route("/funds/disclosure-holders-search", search)
assert client.funds.search_disclosure_holders(FEDERATED)[0].entity_org_type is None

search[0]["entityOrgType"] = "SECRET"
fixture_server.route("/funds/disclosure-holders-search", search)
with pytest.raises(errors.FmpDecodeError) as raised:
client.funds.search_disclosure_holders(FEDERATED)
assert raised.value.endpoint == "funds/disclosure-holders-search"
assert raised.value.decode_path == "[0].entityOrgType"
assert raised.value.decode_kind == "invalid_value"


def test_search_fund_disclosure_holders_with_a_plain_name(client: Any, fixture_server: FixtureServer) -> None:
"""``search_disclosure_holders`` sends a space-separated name as ``+`` and nothing else."""
fixture_server.route("/funds/disclosure-holders-search", load_fixture("fund_disclosure_holder_search.json"))
Expand Down
71 changes: 71 additions & 0 deletions crates/libfmp/src/codecs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -807,6 +807,33 @@ pub(crate) mod empty_or_null {
}
}

/// Deserializer for a required wire key holding a [`NumericString`] whose
/// absent value the provider sends as the text `"NULL"` (ADR 0033, #380).
///
/// The exact text `"NULL"` and JSON null decode to `None`; any other string
/// must be a numeric string, so `""`, `"null"` or `" NULL"` are still decode
/// errors, and that error never includes the value. The key stays required:
/// do not add `default` at call sites. Apply with
/// `deserialize_with = "crate::codecs::null_text::deserialize"` (the exact
/// path gen_go matches); serialization is serde's default, so `None`
/// re-encodes as null.
pub(crate) mod null_text {
use super::*;

pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<Option<NumericString>, D::Error>
where
D: Deserializer<'de>,
{
match Option::<String>::deserialize(deserializer)? {
None => Ok(None),
Some(value) if value == "NULL" => Ok(None),
Some(value) => NumericString::new(value)
.map(Some)
.map_err(de::Error::custom),
}
}
}

/// Serializers for response fields typed with an integral-`f64` alias.
///
/// The provider documents fields such as [`Volume`](crate::types::Volume)
Expand Down Expand Up @@ -967,6 +994,50 @@ mod tests {
);
}

#[test]
fn null_text_maps_the_null_text_and_null_to_none_and_parses_the_rest() {
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct Row {
#[serde(deserialize_with = "super::null_text::deserialize")]
org_type: Option<super::NumericString>,
}
let decode = |text: &str| serde_json::from_str::<Row>(text);
let from_value =
|text: &str| serde_json::from_value::<Row>(serde_json::from_str(text).unwrap());

for text in [r#"{"org_type":"NULL"}"#, r#"{"org_type":null}"#] {
assert_eq!(decode(text).unwrap(), Row { org_type: None });
assert_eq!(from_value(text).unwrap(), Row { org_type: None });
}
let row = decode(r#"{"org_type":"30"}"#).unwrap();
assert_eq!(
row.org_type.as_ref().map(|value| value.as_str()),
Some("30")
);
assert_eq!(serde_json::to_string(&row).unwrap(), r#"{"org_type":"30"}"#);
assert_eq!(
serde_json::to_string(&Row { org_type: None }).unwrap(),
r#"{"org_type":null}"#
);

for (text, secret) in [
(r#"{"org_type":"SECRET"}"#, "SECRET"),
(r#"{"org_type":"null"}"#, "null"),
(r#"{"org_type":" NULL"}"#, "NULL"),
(r#"{"org_type":""}"#, "\"\""),
] {
let error = decode(text).unwrap_err().to_string();
assert!(from_value(text).is_err(), "from_value {text}");
assert!(
!error.contains(secret),
"the error must not echo the value: {error}"
);
}
assert!(decode(r#"{"org_type":30}"#).is_err());
let missing = decode("{}").unwrap_err().to_string();
assert!(missing.contains("missing field `org_type`"), "{missing}");
}

#[test]
fn count_accepts_integers_and_integral_floats_on_both_decode_paths() {
#[derive(Debug, PartialEq, Serialize, Deserialize)]
Expand Down
3 changes: 2 additions & 1 deletion crates/libfmp/src/responses/funds.rs
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,8 @@ pub struct FundDisclosureSearchResult {
pub class_id: String,
pub series_id: String,
pub entity_name: String,
pub entity_org_type: NumericString,
#[serde(deserialize_with = "crate::codecs::null_text::deserialize")]
pub entity_org_type: Option<NumericString>,
pub series_name: String,
pub class_name: String,
pub reporting_file_number: String,
Expand Down
2 changes: 1 addition & 1 deletion crates/libfmp/tests/fund_disclosure_endpoints.rs
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ async fn custom_proxy_preserves_exact_query_order_encoding_headers_and_fixture_i
ApiDateTime::parse("2023-12-28 09:26:13").unwrap()
);
assert_eq!(search[0].cik.as_str(), "0000355691");
assert_eq!(search[0].entity_org_type.as_str(), "30");
assert_eq!(search[0].entity_org_type.as_ref().unwrap().as_str(), "30");
assert_eq!(dates[0].year.get(), 2026);
assert_eq!(dates[0].quarter.get(), 2);

Expand Down
21 changes: 19 additions & 2 deletions crates/libfmp/tests/funds_responses.rs
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,10 @@ fn exact_search_fixture_preserves_all_thirteen_string_fields() {
row.entity_name,
"Federated Hermes Government Income Securities, Inc."
);
assert_eq!(row.entity_org_type.as_str(), "30");
assert_eq!(
row.entity_org_type.as_ref().map(NumericString::as_str),
Some("30")
);
assert_eq!(
row.series_name,
"Federated Hermes Government Income Securities, Inc."
Expand Down Expand Up @@ -346,7 +349,7 @@ fn all_fields_and_nested_sector_fields_are_required_non_null_and_forward_toleran
assert_contract::<EtfSectorWeighting>(SECTOR, &[]);
assert_contract::<FundDisclosureHolder>(LATEST_HOLDERS, &[]);
assert_contract::<FundDisclosure>(DISCLOSURES, &["symbol", "isin"]);
assert_contract::<FundDisclosureSearchResult>(SEARCH, &["address"]);
assert_contract::<FundDisclosureSearchResult>(SEARCH, &["entityOrgType", "address"]);
assert_contract::<FundDisclosureDate>(DATES, &[]);

for field in ["industry", "exposure"] {
Expand Down Expand Up @@ -432,6 +435,20 @@ fn omittable_members_decode_null_and_empty_as_none() {
search[0]["address"] = serde_json::json!("");
let rows: Vec<FundDisclosureSearchResult> = serde_json::from_value(search).unwrap();
assert_eq!(rows[0].address.as_deref(), Some(""));

let mut search: serde_json::Value = serde_json::from_slice(SEARCH).unwrap();
search[0]["entityOrgType"] = serde_json::json!("NULL");
let rows: Vec<FundDisclosureSearchResult> = serde_json::from_value(search).unwrap();
assert_eq!(rows[0].entity_org_type, None);
assert!(serde_json::to_value(&rows).unwrap()[0]["entityOrgType"].is_null());
for wire in ["SECRET", "null", ""] {
let mut search: serde_json::Value = serde_json::from_slice(SEARCH).unwrap();
search[0]["entityOrgType"] = serde_json::json!(wire);
let error = serde_json::from_value::<Vec<FundDisclosureSearchResult>>(search)
.unwrap_err()
.to_string();
assert!(!error.contains("SECRET"), "{error}");
}
}

fn source_row(fixture: &[u8]) -> serde_json::Value {
Expand Down
14 changes: 11 additions & 3 deletions docs/adr/0033-nullable-response-members.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ evidence is now a second source, so both need a rule.
| `""` on a typed code (`CountryCode`, `Isin`, `Ticker`, ...) | `Option<T>` through `codecs::empty_or_null` | `None` | `nil` |
| `""` on a date | `Option<Date>` through `codecs::empty_or_null_date` | `None` | `nil` |
| `""` on a plain `String` | unchanged: stays `""` | `""` | `""` |
| `"NULL"` text on a `NumericString` (`FundDisclosureSearchResult.entityOrgType`, #380) | `Option<NumericString>` through `codecs::null_text` | `None` | `nil` |

- Evidence decides: a member changes only when the audit (or a later
report) saw it null or empty. Parity siblings are not widened.
Expand All @@ -37,8 +38,14 @@ evidence is now a second source, so both need a rule.
value. `None` re-encodes as `null`.
- No invented values: a null amount never becomes `0` or `NaN`, and a row
is never dropped to hide a bad member.
- `codecs::null_text` accepts the exact text `"NULL"` and null as `None`
and parses any other string as a `NumericString`, so `""`, `"null"`, or a
padded `" NULL"` is still an error that never includes the value. It is
used only where the evidence shows `"NULL"` standing in for an absent
value (the live rows that send it on `entityOrgType` also send a null
`address`); no other codec treats `"NULL"` as absent.
- A member whose non-null value has the wrong type (a year where a date is
documented, the text `"NULL"`) is a separate type fix, not an `Option`.
documented) is a separate type fix, not an `Option`.
- Every change is breaking and ships in a minor release with a
`BREAKING CHANGE:` footer naming the members.

Expand All @@ -47,8 +54,9 @@ evidence is now a second source, so both need a rule.
`gen_go` maps `deserialize_with = "crate::codecs::empty_or_null::deserialize"`
on an `Option` of a string-backed type to `Codec::EmptyOrNullString`: the
shadow member keeps the raw `jsontext.Value`, the key is required, and
`""` or null decode to `nil`. It fails on any other shape rather than
guessing.
`""` or null decode to `nil`. `crate::codecs::null_text::deserialize` maps
the same way to `Codec::NullTextString`, where the text `"NULL"` or null
decode to `nil`. It fails on any other shape rather than guessing.

## Alternatives considered

Expand Down
Loading
Loading