From 97b94bc4aefb842ae01c94b61e56664b36030d15 Mon Sep 17 00:00:00 2001 From: Ramon DALMAU Date: Thu, 22 Oct 2026 06:26:55 +0200 Subject: [PATCH 1/3] fix(fuel): reject a q_fuel outside 35-50 MJ/kg (unit error guard) q_fuel must be in J/kg. A value in MJ/kg (e.g. 43.05) made the BADA fuel flow 1e6 times too large: every flight using it failed with "BADA fuel flow is unrealistic", which does not point to the fuel value. When fuel flow, efficiency and mass are all provided, the value reached SAC/CoCiP silently. - check_q_fuel (neats_fuel.py): a supplied q_fuel must lie within Q_FUEL_MIN-Q_FUEL_MAX (35e6-50e6 J/kg); NEATSFuel refuses others. - Parser: an out-of-range q_fuel fails the flight at parsing, naming its source (AO calorific_value, or the airport fuel file and airport). - Airport fuel file: checked at load, outside the block that turns a missing or unreadable file into a warning, and when FleetRunnerParams is validated (fleet steps are built lazily in the workers). A wrong file stops the run, naming the airports. A missing file still warns. Valid inputs are unchanged. Co-Authored-By: Claude Opus 5.5 --- src/pyneats/core/neats_default_parameters.py | 11 ++ src/pyneats/core/neats_fuel.py | 31 ++++- src/pyneats/runners/fleet.py | 7 + src/pyneats/steps/parsing/neats_parser.py | 47 ++++++- tests/test_q_fuel_plausibility.py | 135 +++++++++++++++++++ 5 files changed, 227 insertions(+), 4 deletions(-) create mode 100644 tests/test_q_fuel_plausibility.py diff --git a/src/pyneats/core/neats_default_parameters.py b/src/pyneats/core/neats_default_parameters.py index e5051fe..05b48e1 100644 --- a/src/pyneats/core/neats_default_parameters.py +++ b/src/pyneats/core/neats_default_parameters.py @@ -63,6 +63,9 @@ Reference q_fuel value, in J/kg (pycontrails JetA lower calorific value). DEFAULT_Q_FUEL : float Default q_fuel value, in J/kg (lower calorific value). +Q_FUEL_MIN, Q_FUEL_MAX : float + Plausible range of a supplied q_fuel, in J/kg (35-50 MJ/kg). A value outside + it is rejected, which catches a value given in MJ/kg instead of J/kg. OPEN_AIRCLIM_LEVELS : tuple of int Default pressure levels for Open Air Clim, in hPa. OPEN_AIRCLIM_GRID_RES : float @@ -91,6 +94,8 @@ "DEFAULT_INTERPOLATION_TIME", "DEFAULT_BADA4_VERSION", "DEFAULT_Q_FUEL", + "Q_FUEL_MIN", + "Q_FUEL_MAX", "DEFAULT_DELTA_TAU_COMPUTE_METHOD", "DEFAULT_DELTA_TAU_FILL_METHOD", "DEFAULT_TRUE_AIR_SPEED_SMOOTHING_WINDOW", @@ -334,6 +339,12 @@ DEFAULT_Q_FUEL: Final[float] = ( 42_800_000.0 # J/kg - Default value provided by consortium (DLR/TO70) ) +# Plausible range of a supplied q_fuel (AO calorific_value or airport fuel file). +# Jet fuels are about 42.8-43.5 MJ/kg; the range is wide on purpose and only +# catches unit errors: a value in MJ/kg (e.g. 43.05) is 1e6 times too small and +# would make the BADA fuel flow 1e6 times too large. +Q_FUEL_MIN: Final[float] = 35_000_000.0 # J/kg +Q_FUEL_MAX: Final[float] = 50_000_000.0 # J/kg # OpenAirClim Parameters diff --git a/src/pyneats/core/neats_fuel.py b/src/pyneats/core/neats_fuel.py index 247a22c..4a8f174 100644 --- a/src/pyneats/core/neats_fuel.py +++ b/src/pyneats/core/neats_fuel.py @@ -19,6 +19,8 @@ from pyneats.core.neats_default_parameters import DEFAULT_NAPHTHALEN_CONTENT from pyneats.core.neats_default_parameters import DEFAULT_Q_FUEL from pyneats.core.neats_default_parameters import DEFAULT_SULPHUR_CONTENT +from pyneats.core.neats_default_parameters import Q_FUEL_MAX +from pyneats.core.neats_default_parameters import Q_FUEL_MIN __all__ = [ @@ -26,6 +28,29 @@ ] +def check_q_fuel(value: Any, source: str) -> float: + """Return ``value`` as a float in J/kg, or raise if it is not plausible. + + A supplied q_fuel must lie within ``Q_FUEL_MIN``-``Q_FUEL_MAX`` (35-50 MJ/kg, + in J/kg). The check exists to catch unit errors: a value in MJ/kg (e.g. + 43.05) is 1e6 times too small and would make the BADA fuel flow 1e6 times + too large. ``source`` names where the value came from, for the message. + + Raises + ------ + ValueError + If ``value`` is not a number, or is NaN or outside the range. + """ + q = float(value) + if not Q_FUEL_MIN <= q <= Q_FUEL_MAX: + raise ValueError( + f"q_fuel = {q:g} J/kg from {source} is outside the plausible range " + f"{Q_FUEL_MIN / 1e6:g}-{Q_FUEL_MAX / 1e6:g} MJ/kg. q_fuel must be in " + f"J/kg: multiply a value in MJ/kg by 1e6." + ) + return q + + class NEATSFuel(SAFBlend): """ SAF-like type (passes isinstance(..., SAFBlend) and truthy pct_blend) with @@ -47,7 +72,9 @@ class NEATSFuel(SAFBlend): The AO JSON field ``calorific_value`` is in MJ/kg of fuel. The NEATS application multiplies by 1e6 before calling PyNEATS. Values received here must already be in J/kg; do not convert again. - Defaults to ``DEFAULT_Q_FUEL`` (42.8e6 J/kg). + Defaults to ``DEFAULT_Q_FUEL`` (42.8e6 J/kg). A supplied value outside + ``Q_FUEL_MIN``-``Q_FUEL_MAX`` (35-50 MJ/kg) raises ``ValueError`` + (see ``check_q_fuel``). pct_blend_gate : float, optional Blend gate for SAFBlend compatibility. sulphur_content : float, optional @@ -118,7 +145,7 @@ def __init__( h_wt_pct = DEFAULT_HYDROGEN_CONTENT # --- Resolve q_fuel [J/kg, LHV]; already in J/kg on entry (see docstring) - qf = float(q_fuel) if q_fuel is not None else DEFAULT_Q_FUEL + qf = check_q_fuel(q_fuel, "q_fuel") if q_fuel is not None else DEFAULT_Q_FUEL # --- Storing Optional Fuel Attributes from AOs, not used at this stage if aromatics_content is not None: diff --git a/src/pyneats/runners/fleet.py b/src/pyneats/runners/fleet.py index 6dd7569..fba1cb4 100644 --- a/src/pyneats/runners/fleet.py +++ b/src/pyneats/runners/fleet.py @@ -63,6 +63,7 @@ from pyneats.steps.parsing.neats_io import FlightConversionFailure from pyneats.steps.parsing.neats_io import neats_json_to_flights from pyneats.steps.parsing.neats_io import split_df_into_flights +from pyneats.steps.parsing.neats_parser import _load_airport_fuel_props from pyneats.steps.parsing.protocol import TrajectoryParser from pyneats.steps.parsing.views import Flight4D from pyneats.steps.performance import PerformanceModel @@ -214,6 +215,12 @@ def validate(self) -> None: "Must provide either trajectory_json_filepath or trajectory_dataframe" ) + # Check the airport fuel file here, at construction: the parser is built + # lazily in each worker, so a wrong file would otherwise fail every + # flight that uses it instead of stopping the run. + if self.airport_fuel_path is not None: + _load_airport_fuel_props(self.airport_fuel_path) + # ----------------------------------------------------------------------------- # Generic step cache (per-process) built on steps_registry.build() diff --git a/src/pyneats/steps/parsing/neats_parser.py b/src/pyneats/steps/parsing/neats_parser.py index a53ff46..e5dd7e6 100644 --- a/src/pyneats/steps/parsing/neats_parser.py +++ b/src/pyneats/steps/parsing/neats_parser.py @@ -15,6 +15,7 @@ from pycontrails.physics.units import ft_to_m from pyneats.core.neats_fuel import NEATSFuel +from pyneats.core.neats_fuel import check_q_fuel from pyneats.core.steps import BaseStep from pyneats.core.steps_registry import register from pyneats.core.views import ValidationError @@ -50,7 +51,36 @@ def _load_airport_fuel_props( path: str | None, ) -> dict[str, dict[str, float]]: - """Load airport → {attr_name: value} mapping from a CSV or JSON file. + """Load the airport fuel file (see ``_read_airport_fuel_props``) and check it. + + Every ``q_fuel`` must lie within 35-50 MJ/kg, given in J/kg (``check_q_fuel``). + A value outside the range, typically a value in MJ/kg, raises ``ValueError`` + naming the airports, so a wrong file stops the run instead of failing every + flight that departs from those airports. A missing or unreadable file still + gives ``{}`` with a warning. + """ + props = _read_airport_fuel_props(path) + bad: list[str] = [] + for airport, values in props.items(): + if "q_fuel" in values: + try: + check_q_fuel(values["q_fuel"], f"airport {airport}") + except ValueError: + bad.append(f"{airport}={values['q_fuel']:g}") + if bad: + raise ValueError( + f"Airport fuel file {path}: q_fuel outside the plausible range 35-50 MJ/kg " + f"for {len(bad)} airport(s) ({', '.join(bad[:5])}" + f"{', ...' if len(bad) > 5 else ''}). q_fuel must be in J/kg: multiply " + f"a value in MJ/kg by 1e6." + ) + return props + + +def _read_airport_fuel_props( + path: str | None, +) -> dict[str, dict[str, float]]: + """Read airport → {attr_name: value} mapping from a CSV or JSON file. Supports: - Semicolon-delimited CSV with European decimal commas (Aviation_Fuel_Reports format) @@ -306,12 +336,17 @@ def run(self, flight: pd.DataFrame) -> Flight4D: # 5.5) Airport-based fuel-property fallback (operator values always win) # Values are used as-is (no unit conversion; q_fuel in J/kg). + q_fuel_source = "the flight attrs (AO calorific_value)" if self._airport_fuel_props: airport = str(attrs.get("departure_airport", "")).upper() if airport in self._airport_fuel_props: for attr_key, val in self._airport_fuel_props[airport].items(): if attrs.get(attr_key) is None: attrs[attr_key] = val + if attr_key == "q_fuel": + q_fuel_source = ( + f"the airport fuel file (airport {airport})" + ) self.logger.debug( "Airport fuel prop %s for %s: %s", attr_key, @@ -326,7 +361,15 @@ def run(self, flight: pd.DataFrame) -> Flight4D: "processed with FleetRunner, not FlightRunner." ) - # 7) Construct Custom Fuel Object based on available attributes + # 7) Construct Custom Fuel Object based on available attributes. + # A supplied q_fuel outside 35-50 MJ/kg fails the flight here, + # with its source named (usually a value in MJ/kg, not J/kg). + q_fuel = NEATSFuel._to_scalar(attrs.get("q_fuel")) + if q_fuel is not None: + try: + check_q_fuel(q_fuel, q_fuel_source) + except ValueError as e: + raise TrajectoryParserStepError(str(e)) from e fuel_obj: NEATSFuel = NEATSFuel.from_attrs(attrs) # 8) Construct base Flight with required + optional columns only diff --git a/tests/test_q_fuel_plausibility.py b/tests/test_q_fuel_plausibility.py new file mode 100644 index 0000000..6a04442 --- /dev/null +++ b/tests/test_q_fuel_plausibility.py @@ -0,0 +1,135 @@ +"""A supplied q_fuel outside 35-50 MJ/kg is rejected (audit A10). + +q_fuel must be in J/kg. A value in MJ/kg (e.g. 43.05) is 1e6 times too small and +made the BADA fuel flow 1e6 times too large: every flight using it failed with +"BADA fuel flow is unrealistic", which does not point to the fuel value. + +Now: +- an airport fuel file with such a value stops the run when the runner or the + parser is built, naming the airports; +- a flight with such a value fails at parsing, naming the source (AO value or + airport file); +- ``NEATSFuel`` refuses it. +""" + +from __future__ import annotations + +from pathlib import Path + +import pandas as pd +import pytest + +from pyneats.core.neats_default_parameters import DEFAULT_Q_FUEL +from pyneats.core.neats_default_parameters import Q_FUEL_MAX +from pyneats.core.neats_default_parameters import Q_FUEL_MIN +from pyneats.core.neats_fuel import NEATSFuel +from pyneats.core.neats_fuel import check_q_fuel +from pyneats.runners.fleet import FleetRunnerParams +from pyneats.steps.parsing.neats_parser import NeatsTrajectoryParser +from pyneats.steps.parsing.neats_parser import _load_airport_fuel_props +from pyneats.steps.parsing.protocol import TrajectoryParserStepError +from pyneats.steps.weather.weather_store import ZarrPaths + + +REQUIRED_ATTRS = { + "flight_id": "FL001", + "aircraft_type": "A320", + "departure_airport": "EGLL", + "arrival_airport": "LFPG", + "model_type": "BADA4", + "aobt": "2023-01-01 10:00:00", +} + + +def _flight_df(**fuel_attrs: float) -> pd.DataFrame: + df = pd.DataFrame( + { + "latitude": [51.5, 51.6], + "longitude": [-0.1, 0.0], + "altitude": [100, 200], + "time": ["2023-01-01 10:00:00", "2023-01-01 10:05:00"], + } + ) + df.attrs = {**REQUIRED_ATTRS, **fuel_attrs} + return df + + +def _airport_csv(tmp_path: Path, rows: dict[str, str]) -> str: + # Same layout as the production file: ';' separator, decimal comma. + p = tmp_path / "airport_fuel.csv" + lines = ["Airport;aromatics_content;h_c_ratio;naphthalene;sulphur_content;q_fuel"] + lines += [f"{a};0,2;;0,01;0,0001;{q}" for a, q in rows.items()] + p.write_text("\n".join(lines), encoding="utf-8") + return str(p) + + +@pytest.mark.parametrize( + "q", [Q_FUEL_MIN, Q_FUEL_MAX, DEFAULT_Q_FUEL, 41.944e6, 43.13e6, 45.5e6] +) +def test_plausible_q_fuel_accepted(q: float) -> None: + assert check_q_fuel(q, "test") == q + assert NEATSFuel(q_fuel=q).q_fuel == q + + +@pytest.mark.parametrize("q", [43.05, 43_050.0, 34.9e6, 50.1e6, float("nan")]) +def test_implausible_q_fuel_rejected(q: float) -> None: + with pytest.raises(ValueError, match="must be in J/kg"): + check_q_fuel(q, "test") + with pytest.raises(ValueError, match="outside the plausible range"): + NEATSFuel(q_fuel=q) + + +def test_default_q_fuel_unchanged() -> None: + assert NEATSFuel().q_fuel == DEFAULT_Q_FUEL + + +def test_airport_file_in_mj_per_kg_rejected_at_load(tmp_path: Path) -> None: + path = _airport_csv(tmp_path, {"EGLL": "43000000", "EKYT": "43,05"}) + with pytest.raises(ValueError, match=r"1 airport\(s\) \(EKYT=43.05\)"): + _load_airport_fuel_props(path) + + +def test_airport_file_in_j_per_kg_loads(tmp_path: Path) -> None: + path = _airport_csv(tmp_path, {"EGLL": "43000000", "EKYT": "43050000"}) + props = _load_airport_fuel_props(path) + assert props["EKYT"]["q_fuel"] == 43_050_000.0 + + +def test_parser_with_wrong_airport_file_fails_at_construction(tmp_path: Path) -> None: + path = _airport_csv(tmp_path, {"EGLL": "43,0"}) + with pytest.raises(ValueError, match="EGLL"): + NeatsTrajectoryParser(airport_fuel_path=path) + + +def test_fleet_runner_params_check_airport_file(tmp_path: Path) -> None: + # FleetRunner builds the parser lazily in each worker, so the file is + # checked when the configuration is validated (FleetRunner.__init__). + params = FleetRunnerParams( + zarr_paths=ZarrPaths(str(tmp_path / "met"), str(tmp_path / "rad")), + trajectory_json_filepath=str(tmp_path / "flights.json"), + airport_fuel_path=_airport_csv(tmp_path, {"EGLL": "43,0"}), + ) + with pytest.raises(ValueError, match="EGLL"): + params.validate() + + +def test_missing_airport_file_still_only_warns(tmp_path: Path) -> None: + assert _load_airport_fuel_props(str(tmp_path / "missing.csv")) == {} + + +def test_flight_with_ao_q_fuel_in_mj_per_kg_fails_parsing() -> None: + with pytest.raises(TrajectoryParserStepError, match="AO calorific_value"): + NeatsTrajectoryParser().run(_flight_df(q_fuel=43.05)) + + +def test_flight_with_airport_q_fuel_out_of_range_names_airport() -> None: + # A file that bypassed the load check (e.g. a mapping set directly). + parser = NeatsTrajectoryParser() + parser._airport_fuel_props = {"EGLL": {"q_fuel": 43.05}} + with pytest.raises(TrajectoryParserStepError, match="airport EGLL"): + parser.run(_flight_df()) + + +def test_flight_with_plausible_q_fuel_parses() -> None: + flight = NeatsTrajectoryParser().run(_flight_df(q_fuel=43.05e6)) + assert flight.fuel.q_fuel == 43.05e6 From 229ac2b4f8409284d4baa9a6d88b07d292c135f3 Mon Sep 17 00:00:00 2001 From: Ramon DALMAU Date: Thu, 22 Oct 2026 06:26:57 +0200 Subject: [PATCH 2/3] docs: fuel properties units, airport fuel file format and the q_fuel range - input-format.md: fuel_properties units (hydrogen_content is wt.%, not a mass fraction; calorific_value J/kg, 35e6-50e6), which fields are used, and a section on the airport fuel file (format, units, precedence, checks). - input-prioritization.md and mrv-specification.md: the airport file in the fuel-property fallback, and the q_fuel range. - Units audit: A10 fixed, with the follow-up (production file read, all runner paths checked on dev); status row. Co-Authored-By: Claude Opus 5.5 --- docs/explanation/input-prioritization.md | 2 +- docs/explanation/mrv-specification.md | 3 +- docs/internal/pyneats_units_audit.md | 39 +++++++++++++----- docs/reference/input-format.md | 52 ++++++++++++++++++++---- 4 files changed, 75 insertions(+), 21 deletions(-) diff --git a/docs/explanation/input-prioritization.md b/docs/explanation/input-prioritization.md index 84c4469..8e92111 100644 --- a/docs/explanation/input-prioritization.md +++ b/docs/explanation/input-prioritization.md @@ -15,7 +15,7 @@ This page describes the decision logic for determining which performance and fue | Engine efficiency | Recorded η | From BADA thrust and BADA fuel flow | | Aircraft mass | Mass evolution | Take-off weight decreased along the trajectory with the BADA fuel flow | | Take-off weight | AO-declared TOW | Iterative estimate: min(MTOW, OEW + load_factor × MPL + 1.15 × trip fuel), see [Mass Estimation](#mass-estimation) | -| Fuel properties | AO-declared (H/C, q_fuel) | JET-A defaults | +| Fuel properties | AO-declared (H/C, q_fuel) | Airport fuel file for the departure airport (if configured), then the consortium defaults (q_fuel 42.8 MJ/kg, hydrogen 13.79 wt.%) | | Engine UID | AO-declared | MRR conservative value | --- diff --git a/docs/explanation/mrv-specification.md b/docs/explanation/mrv-specification.md index be9625d..8551281 100644 --- a/docs/explanation/mrv-specification.md +++ b/docs/explanation/mrv-specification.md @@ -245,7 +245,8 @@ To account for fuel properties provided by AOs, a **custom `NEATSFuel` class** h $$\text{Hydrogen content} = \frac{r \cdot 1.008}{12.011 + r \cdot 1.008}$$ - - If the AO provides the **calorific value**, it overrides the `q_fuel` conservative default of **42 800 000 J/kg**. + - If the AO provides the **calorific value**, it overrides the `q_fuel` conservative default of **42 800 000 J/kg**. If it does not, and an airport fuel file is configured, the value for the departure airport is used. + - A supplied `q_fuel` must be in J/kg and between **35 and 50 MJ/kg**. Outside this range (typically a value in MJ/kg) the flight fails at parsing; a wrong value in the airport fuel file stops the run when the runner is built. - If **neither** hydrogen content **nor** hydrogen-to-carbon ratio is provided, the hydrogen content defaults to the conservative **13.79 %**. - The water emission index **`ei_h2o`** (initial default 1.23 in PyContrails/CoCiP) is **adjusted linearly** based on the hydrogen content value, in line with what is done in `SAFBlend`. - The **nvPM** is reduced using the `black_carbon` class from PyContrails — as done in `SAFBlend` — taking the hydrogen content as parameter (currently only available for SAF in PyContrails). diff --git a/docs/internal/pyneats_units_audit.md b/docs/internal/pyneats_units_audit.md index d0a18e5..1b45dd9 100644 --- a/docs/internal/pyneats_units_audit.md +++ b/docs/internal/pyneats_units_audit.md @@ -21,6 +21,7 @@ cite line numbers on `dev` before the fix they describe. | A13 (mixed fuel-flow provenance) | Documented in the user docs; the design choice stays open | #75 | 2026-10-21 | | I9 (`segment_duration` not recomputed after the speed filter) | Open; recorded, part of the low-speed filter work | #75 | 2026-10-21 | | I10 (a failed attempt leaked columns into the BADA3 retry) | Fixed; fuel-burn rejections go through the BADA3 retry and the altitude fallback (maintainer decision); no change on the NEATS_FINAL sample | `fix/performance-retry-isolation` | 2026-10-21 | +| A10 (airport fuel file units) and the missing `q_fuel` magnitude check | Fixed: a supplied `q_fuel` outside 35–50 MJ/kg is rejected (flight fails; a wrong airport file stops the run); no change on valid inputs | `fix/q-fuel-plausibility` | 2026-10-22 | ## 1. Outcome @@ -165,7 +166,7 @@ golden data. *Stated* means written in a comment, docstring or document only. | Mapping | Convention | PyNEATS boundary unit | Internal or output unit | Existing conversion and layer | Status | Evidence | | --- | --- | --- | --- | --- | --- | --- | | DataFrame entry (`trajectory_dataframe`, `FlightRunner(source=df)`): `altitude`, `time` | Not defined by AO upload schema | `altitude` in **FL**, not metres. `time` as `datetime64` (naive means UTC; tz-aware is converted) or a string in `date_format` (default `%Y-%m-%d %H:%M:%S`) | as for the JSON path | the same parser conversions | **Ambiguous / unverified** (A9) | `neats_parser.py:141,217-258`; example comment `examples/fleet_computation_from_dataframe.py:172` *(stated)*; snippet: ISO strings become NaT and are dropped *(demonstrated)* | -| Airport fuel-property file (`airport_fuel_path`) | Not defined by AO upload schema | `float()` only. `q_fuel` assumed J/kg; `h_c_ratio` used as atomic | J/kg, wt.% after `NEATSFuel` | **none**. PyNEATS reads the file directly, so the application's × 1e6 does not apply to it | **Ambiguous / unverified** (A10) | `neats_parser.py:35-41,44-120,284-296`; `tests/test_airport_qfuel.py` (J/kg) | +| Airport fuel-property file (`airport_fuel_path`) | Not defined by AO upload schema | `float()` only. `q_fuel` assumed J/kg; `h_c_ratio` used as atomic | J/kg, wt.% after `NEATSFuel` | **none**. PyNEATS reads the file directly, so the application's × 1e6 does not apply to it | **Checked**: 35–50 MJ/kg in J/kg enforced at load (A10, fixed) | `neats_parser.py:35-41,44-120,284-296`; `tests/test_airport_qfuel.py` (J/kg) | | Default branch: no `calorific_value` | Not defined by AO upload schema | `DEFAULT_Q_FUEL` = 42.8e6 J/kg | the BADA fuel flow is still multiplied by 43.13/42.8 = 1.0077 | correction ratio in `BADAPerformanceModel` | **Ambiguous / unverified** (A11) | `neats_fuel.py:92`; `bada_model.py:565-571,764-772,841,875`; `docs/explanation/mrv-specification.md:220` *(stated)* | | `REFERENCE_Q_FUEL` against BADA4 LHV | Not defined by AO upload schema | 43.13e6 J/kg (pycontrails JetA) | — | not read from BADA files, although BADA4 carries a per-aircraft LHV | **Ambiguous / unverified** (A12) | `neats_default_parameters.py:315-317`; pyBADA `bada4.py:1855-1874`; pyBADA dummy BADA4 files: 43.217e6 and 43.71e6 J/kg | | Per-point nulls in `tas`, `ff`, `ee`, `am` | schema allows `null` per point | a column survives unless every value is null | NaN spreads through interpolation and is skipped by sums | none | **New discrepancy** (D4) | `neats_io.py:93-94`; `steps/interpolation/pycontrails_interpolation.py:108-112`; `bada_model.py:1071-1076`; snippet *(demonstrated)* | @@ -215,9 +216,11 @@ JSON path, `scripts/run_json.py` and DataFrame users. Two places get no upstream - **Airport fuel file.** PyNEATS reads `airport_fuel_path` directly, so a `q_fuel` column in MJ/kg would enter unconverted (A10). -- **No magnitude check.** PyNEATS does not check the size of `q_fuel`. On the BADA path, - 42.8 (MJ/kg) fails loudly through the 100 kg/s fuel-flow outlier check. When `ff`, `ee` and - `am` are all provided, BADA is skipped and the value reaches SAC/CoCiP silently. +- **No magnitude check.** PyNEATS did not check the size of `q_fuel`. On the BADA path, + 42.8 (MJ/kg) failed through the 100 kg/s fuel-flow outlier check with a message that does + not name the fuel value. When `ff`, `ee` and `am` are all provided, BADA is skipped and the + value reached SAC/CoCiP silently. *Fixed (see A10): a supplied `q_fuel` outside 35–50 MJ/kg + now fails the flight at parsing, on every path.* `scripts/schemas/flight_schema.json:106-109` labels `calorific_value` as MJ/kg. The golden inputs it validates contain J/kg values, and the code expects J/kg (Section 9). @@ -404,17 +407,33 @@ Each entry gives: 4. Not applicable (direct library use). 5. Documented in the patch. Optionally rename the column or check its range. -**A10. Airport fuel file units.** -1. `_load_airport_fuel_props` (`neats_parser.py:44-120`) → fallback (`neats_parser.py:284-296`). +**A10. Airport fuel file units.** *Fixed on branch `fix/q-fuel-plausibility` (see the +follow-up status table).* +1. `_load_airport_fuel_props` (`neats_parser.py`) → fallback in the parser (step 5.5). 2. Values are passed through `float()` only. The tests use `q_fuel` in J/kg. `h_c_ratio` is - treated as atomic. There is no `hydrogen_content` column. The file's own units are - unverified, and no production file is in the repository. + treated as atomic. There is no `hydrogen_content` column. 3. A `q_fuel` column in MJ/kg would enter 1e6 times too small, because the application's conversion does not apply to this file. A mass-based `h_c_ratio` would make `ei_h2o` about 10 times too low. 4. No. PyNEATS reads the file itself. -5. **Confirm units with the file owner.** The patch documents the assumed units in the loader - docstring. Optionally add a magnitude guard for `q_fuel`. +5. **Follow-up (2026-10-22).** + - The file used by NEATS (`Airport_Fuel_2025.csv`, 145 airports) was read: `q_fuel` is in + J/kg (121 airports, 42.8–43.4 MJ/kg), `h_c_ratio` is empty, and the other columns are + stored but not used. On `dev` (53bda5f) the file is applied correctly by all six runner + paths (fleet large/small/performance-only from JSON, fleet from a DataFrame, single-flight + large/small): operator value first, then the airport file, then 42.8 MJ/kg. End to end, + fuel changes by 42.8/q as expected (within 0.04 %, the iterative take-off mass). + - Reproduced: with the same file in MJ/kg, every flight using it failed with "BADA fuel + flow is unrealistic". + - **Fix.** A supplied `q_fuel` must lie within `Q_FUEL_MIN`–`Q_FUEL_MAX` (35e6–50e6 J/kg, + `check_q_fuel` in `neats_fuel.py`). The airport file is checked when it is loaded, outside + the block that turns a missing or unreadable file into a warning, and again when + `FleetRunnerParams` is validated, because fleet steps are built lazily in the workers. A + wrong file stops the run, naming the airports. A flight with an out-of-range value fails + at parsing, naming the source (AO value or airport file and airport). `NEATSFuel` refuses + such a value. A missing or unreadable file is still only a warning (unchanged). + - The range only catches unit errors (MJ/kg is 1e6 times off); it does not judge whether a + value is realistic for a fuel. Valid inputs give the same results. **A11. `q_fuel` correction on the default branch.** 1. `bada_model.py:565-571,764-772,841,875`. diff --git a/docs/reference/input-format.md b/docs/reference/input-format.md index 47dc9ef..3d40e32 100644 --- a/docs/reference/input-format.md +++ b/docs/reference/input-format.md @@ -35,15 +35,49 @@ sample: [`tests/data/golden/fleet_5_flights_input.json`](https://github.com/euro ### `fuel_properties` -All four fields are nullable. When `null`, airport-level fallback values are -used if an `airport_fuel_path` is configured on the runner. - -| Field | Type | Unit | -|---|---|---| -| `hydrogen_content` | float \| null | mass fraction (0–1) | -| `hydrogen_per_carbon_ratio` | float \| null | — | -| `aromatic_content` | float \| null | mass fraction (0–1) | -| `calorific_value` | float \| null | J/kg | +All fields are nullable. When `null`, the value from the airport fuel file +(below) is used if an `airport_fuel_path` is configured on the runner, otherwise +the default. + +| Field | Type | Unit | Use | +|---|---|---|---| +| `hydrogen_content` | float \| null | wt.% (percent by mass, e.g. 13.79) | scales `ei_h2o`; default 13.79 | +| `hydrogen_per_carbon_ratio` | float \| null | atomic (molar) H/C ratio | converted to wt.% when `hydrogen_content` is `null` | +| `calorific_value` | float \| null | J/kg (lower calorific value), 35e6–50e6 | default 42.8e6; outside the range the flight fails | +| `aromatic_content` | float \| null | not defined | stored, not used | +| `sulphur` | float \| null | not defined | stored, not used | +| `naphthalene` | float \| null | not defined | not used | + +The AO upload gives `calorific_value` in MJ/kg; the NEATS application multiplies +it by 1e6 before calling PyNEATS. A value still in MJ/kg (e.g. `43.05`) is +outside 35e6–50e6 J/kg, so the flight fails at parsing with a message naming +the unit. + +### Airport fuel file (`airport_fuel_path`) + +Optional file passed to the runners (`airport_fuel_path=`). For a flight whose +`fuel_properties` value is `null`, the value for its `departure_airport` is used. +AO values always take precedence. PyNEATS reads this file itself, so no unit +conversion is applied: values must be in the units below. + +- **CSV**: separator `;` or `,` (detected), decimal comma accepted, a column + `Airport` (ICAO code). Recognised columns: `q_fuel` (J/kg), `h_c_ratio` + (atomic), `aromatics_content`, `naphthalene`, `sulphur_content`. Empty cells + are ignored. +- **JSON**: `{"EGLL": 43000000.0}` (q_fuel only) or + `{"EGLL": {"q_fuel": 43000000.0, "h_c_ratio": 1.9}}`. + +Only `q_fuel` and `h_c_ratio` affect results; the other columns are stored. Every +`q_fuel` must lie between 35e6 and 50e6 J/kg: otherwise the runner refuses the +file when it is built, naming the airports. A missing or unreadable file is +ignored with a warning, and the defaults apply. + +Example (the layout of `Airport_Fuel_2025.csv`): + +```text +Airport;aromatics_content;h_c_ratio;naphthalene;sulphur_content;q_fuel +EKYT;0,2125;;0,0136;0,000035;43050000 +``` ### `trajectory` From 58da562b1e1b38767b45fde3d4884f5e4e68a3c1 Mon Sep 17 00:00:00 2001 From: Ramon DALMAU Date: Thu, 22 Oct 2026 07:23:04 +0200 Subject: [PATCH 3/3] fix(fuel): review follow-ups for the q_fuel check - FleetRunnerParams.validate also checks an airport fuel file passed as params["trajectory_parser"]["airport_fuel_path"]. - A NaN q_fuel is reported as empty, not as a value in MJ/kg. - Tests for both; the missing-file test now checks the warning. - input-format.md: hydrogen content also sets the SAF nvPM reduction. - Audit A10: maintainer decision to keep 35-50 MJ/kg; the DLR range [42.6397, 44.5] MJ/kg (TC_QFUEL_OUT_OF_RANGE) stays a known failure. Co-Authored-By: Claude Opus 5.5 --- docs/internal/pyneats_units_audit.md | 5 ++++ docs/reference/input-format.md | 2 +- src/pyneats/core/neats_fuel.py | 5 ++++ src/pyneats/runners/fleet.py | 8 +++++-- tests/test_q_fuel_plausibility.py | 34 +++++++++++++++++++++++++--- 5 files changed, 48 insertions(+), 6 deletions(-) diff --git a/docs/internal/pyneats_units_audit.md b/docs/internal/pyneats_units_audit.md index 1b45dd9..a2fa419 100644 --- a/docs/internal/pyneats_units_audit.md +++ b/docs/internal/pyneats_units_audit.md @@ -434,6 +434,11 @@ follow-up status table).* such a value. A missing or unreadable file is still only a warning (unchanged). - The range only catches unit errors (MJ/kg is 1e6 times off); it does not judge whether a value is realistic for a fuel. Valid inputs give the same results. + - **Maintainer decision (2026-10-22): keep 35–50 MJ/kg.** The DLR test cases define a + narrower valid range, [42.6397, 44.5] MJ/kg (`TC_QFUEL_OUT_OF_RANGE`, + `tests/test_cases_pipeline.py`, 45.5 MJ/kg expected to fail). It was not adopted, so that + test stays a known DLR failure (strict xfail). The same applies to the hydrogen + ([13.79, 15.5] wt.%) and H/C ([1.9061, 2.1857]) range cases. **A11. `q_fuel` correction on the default branch.** 1. `bada_model.py:565-571,764-772,841,875`. diff --git a/docs/reference/input-format.md b/docs/reference/input-format.md index 3d40e32..0273d23 100644 --- a/docs/reference/input-format.md +++ b/docs/reference/input-format.md @@ -41,7 +41,7 @@ the default. | Field | Type | Unit | Use | |---|---|---|---| -| `hydrogen_content` | float \| null | wt.% (percent by mass, e.g. 13.79) | scales `ei_h2o`; default 13.79 | +| `hydrogen_content` | float \| null | wt.% (percent by mass, e.g. 13.79) | scales `ei_h2o` and sets the SAF nvPM reduction; default 13.79 | | `hydrogen_per_carbon_ratio` | float \| null | atomic (molar) H/C ratio | converted to wt.% when `hydrogen_content` is `null` | | `calorific_value` | float \| null | J/kg (lower calorific value), 35e6–50e6 | default 42.8e6; outside the range the flight fails | | `aromatic_content` | float \| null | not defined | stored, not used | diff --git a/src/pyneats/core/neats_fuel.py b/src/pyneats/core/neats_fuel.py index 4a8f174..41d36a8 100644 --- a/src/pyneats/core/neats_fuel.py +++ b/src/pyneats/core/neats_fuel.py @@ -42,6 +42,11 @@ def check_q_fuel(value: Any, source: str) -> float: If ``value`` is not a number, or is NaN or outside the range. """ q = float(value) + if q != q: # NaN + raise ValueError( + f"q_fuel from {source} is NaN; give a value in J/kg or leave it out " + f"(None) to use the default." + ) if not Q_FUEL_MIN <= q <= Q_FUEL_MAX: raise ValueError( f"q_fuel = {q:g} J/kg from {source} is outside the plausible range " diff --git a/src/pyneats/runners/fleet.py b/src/pyneats/runners/fleet.py index fba1cb4..42d8180 100644 --- a/src/pyneats/runners/fleet.py +++ b/src/pyneats/runners/fleet.py @@ -218,8 +218,12 @@ def validate(self) -> None: # Check the airport fuel file here, at construction: the parser is built # lazily in each worker, so a wrong file would otherwise fail every # flight that uses it instead of stopping the run. - if self.airport_fuel_path is not None: - _load_airport_fuel_props(self.airport_fuel_path) + for path in ( + self.airport_fuel_path, + self.params.get("trajectory_parser", {}).get("airport_fuel_path"), + ): + if path is not None: + _load_airport_fuel_props(path) # ----------------------------------------------------------------------------- diff --git a/tests/test_q_fuel_plausibility.py b/tests/test_q_fuel_plausibility.py index 6a04442..c82a2db 100644 --- a/tests/test_q_fuel_plausibility.py +++ b/tests/test_q_fuel_plausibility.py @@ -14,6 +14,7 @@ from __future__ import annotations +import logging from pathlib import Path import pandas as pd @@ -71,7 +72,12 @@ def test_plausible_q_fuel_accepted(q: float) -> None: assert NEATSFuel(q_fuel=q).q_fuel == q -@pytest.mark.parametrize("q", [43.05, 43_050.0, 34.9e6, 50.1e6, float("nan")]) +def test_nan_q_fuel_rejected_as_empty() -> None: + with pytest.raises(ValueError, match="is NaN"): + check_q_fuel(float("nan"), "test") + + +@pytest.mark.parametrize("q", [43.05, 43_050.0, 34.9e6, 50.1e6]) def test_implausible_q_fuel_rejected(q: float) -> None: with pytest.raises(ValueError, match="must be in J/kg"): check_q_fuel(q, "test") @@ -113,8 +119,30 @@ def test_fleet_runner_params_check_airport_file(tmp_path: Path) -> None: params.validate() -def test_missing_airport_file_still_only_warns(tmp_path: Path) -> None: - assert _load_airport_fuel_props(str(tmp_path / "missing.csv")) == {} +def test_fleet_runner_params_check_airport_file_given_as_parser_param( + tmp_path: Path, +) -> None: + params = FleetRunnerParams( + zarr_paths=ZarrPaths(str(tmp_path / "met"), str(tmp_path / "rad")), + trajectory_json_filepath=str(tmp_path / "flights.json"), + params={ + "trajectory_parser": { + "airport_fuel_path": _airport_csv(tmp_path, {"EGLL": "43,0"}) + } + }, + ) + with pytest.raises(ValueError, match="EGLL"): + params.validate() + + +def test_missing_airport_file_still_only_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger="pyneats.steps.parsing.neats_parser"): + assert _load_airport_fuel_props(str(tmp_path / "missing.csv")) == {} + assert any( + "Failed to load airport fuel properties" in r.message for r in caplog.records + ) def test_flight_with_ao_q_fuel_in_mj_per_kg_fails_parsing() -> None: