From c9edd96bf7809d9a32a3b821d0e9fe674ba93b04 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Fri, 20 Nov 2026 21:53:56 +0100 Subject: [PATCH 1/6] feat: score a model against measured growth and fluxes (experiments) The experiments package reads the flux files of several conditions, fixes the uptake of each condition and solves the model, and combines the carbon-weighted RMSE of the flux and max-growth conditions into one distance. The anaerobic switch, the change of the protein content and the solver call are functions that the caller passes. apply_flux_data_constraints sets the exchange bounds of a model from one condition. --- CHANGELOG.md | 4 + src/raven_toolbox/experiments/__init__.py | 35 ++ src/raven_toolbox/experiments/conditions.py | 210 +++++++++ src/raven_toolbox/experiments/constrain.py | 175 +++++++ src/raven_toolbox/experiments/data.py | 88 ++++ src/raven_toolbox/experiments/distance.py | 248 ++++++++++ src/raven_toolbox/experiments/flux_data.py | 256 +++++++++++ tests/test_experiments_conditions.py | 476 ++++++++++++++++++++ tests/test_experiments_constrain.py | 329 ++++++++++++++ tests/test_experiments_data.py | 66 +++ tests/test_experiments_flux_data.py | 283 ++++++++++++ 11 files changed, 2170 insertions(+) create mode 100644 src/raven_toolbox/experiments/__init__.py create mode 100644 src/raven_toolbox/experiments/conditions.py create mode 100644 src/raven_toolbox/experiments/constrain.py create mode 100644 src/raven_toolbox/experiments/data.py create mode 100644 src/raven_toolbox/experiments/distance.py create mode 100644 src/raven_toolbox/experiments/flux_data.py create mode 100644 tests/test_experiments_conditions.py create mode 100644 tests/test_experiments_constrain.py create mode 100644 tests/test_experiments_data.py create mode 100644 tests/test_experiments_flux_data.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d3ac5c..a249113 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ * `get_model_from_homology` with `complex_policy="drop"` removes each complex that has a subunit without an ortholog, and removes the reaction only when no complex remains (was: the reaction was removed as soon as one subunit had no ortholog). For the rule `(A and B) or (C and D)` without an ortholog of `B`, the draft has `C and D`. * `manipulation.contract_model` is the new name of `remove_duplicate_reactions`, after RAVEN's `contractModel`, and the old name is removed. `contract_model` takes `dist_reverse` (default `True`), the `distReverse` option of `contractModel`: with `False`, reactions with the same metabolites and a different reversibility are duplicates. +### Experiments + +* New: the `experiments` package scores a model against measured growth rates and exchange fluxes. `load_experiment_data` reads `fluxData.tsv`, `maxGrowth.tsv` and `zeroFlux.tsv` from a folder, `load_flux_data` parses one flux file into a `FluxData` (with a per-condition `weight`), `simulate_conditions` fixes the uptake of each condition and solves the model, `compute_carbon_weights` gives each exchange the carbon count of its metabolite, `experiment_distance` combines the carbon-weighted RMSE of flux and max-growth conditions into one value, and `apply_flux_data_constraints` sets the exchange bounds of a model from one condition (loose, or a percentage band). The organism-specific steps, an anaerobic switch and a change of the protein content, and the solver call are passed as functions. + ### Data * The KEGG assets are the `kegg120` release of `toolbox-assets` (was: `kegg118`): the reference model and tables, the taxonomy and the profile HMM libraries of 4246 prokaryotic and 2724 eukaryotic KOs. The baked registry and `data/manifest.json` list them. diff --git a/src/raven_toolbox/experiments/__init__.py b/src/raven_toolbox/experiments/__init__.py new file mode 100644 index 0000000..272f498 --- /dev/null +++ b/src/raven_toolbox/experiments/__init__.py @@ -0,0 +1,35 @@ +"""Experimental data on growth and fluxes, and how a model is scored against it. + +The measurements of several conditions are read from files +(:func:`load_experiment_data`), one condition is set up in the model and +simulated (:func:`simulate_conditions`), and the distance between simulated and +measured values is computed (:func:`experiment_distance`). None of this needs +more than a :class:`cobra.Model`; setting that depends on the organism, such as +an anaerobic switch, is passed as a function. +""" +from .conditions import ConditionSimResult, simulate_conditions +from .constrain import apply_flux_data_constraints +from .data import ExperimentData, load_experiment_data +from .distance import ( + BIOMASS_CARBON_EQUIV, + INFEASIBLE_PENALTY, + compute_carbon_weights, + dataset_rmse, + experiment_distance, +) +from .flux_data import FluxData, load_flux_data + +__all__ = [ + "BIOMASS_CARBON_EQUIV", + "INFEASIBLE_PENALTY", + "ConditionSimResult", + "ExperimentData", + "FluxData", + "apply_flux_data_constraints", + "compute_carbon_weights", + "dataset_rmse", + "experiment_distance", + "load_experiment_data", + "load_flux_data", + "simulate_conditions", +] diff --git a/src/raven_toolbox/experiments/conditions.py b/src/raven_toolbox/experiments/conditions.py new file mode 100644 index 0000000..cf73e9f --- /dev/null +++ b/src/raven_toolbox/experiments/conditions.py @@ -0,0 +1,210 @@ +"""Simulate the measured conditions in a model. + +This module only does the FBA half of the distance: given a model and one +:class:`~raven_toolbox.experiments.flux_data.FluxData` dataset (either +``experiment_data.flux_data`` or ``experiment_data.max_growth``), simulate every +condition and report growth and exchange fluxes. +:mod:`.distance` turns those raw numbers into an RMSE against the +matching experimental values -- kept separate so the RMSE math is +testable with hand-built inputs and no model or FBA at all. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Callable, Optional + +import numpy as np + +if TYPE_CHECKING: + import cobra + + from .flux_data import FluxData + + +@dataclass +class ConditionSimResult: + """One condition's simulated outcome. + + ``feasible=False`` (LP infeasible/unbounded, or no biomass flux) + means ``growth``/``exch_fluxes``/``block_fluxes`` are meaningless; + :mod:`.distance` applies the fixed infeasibility penalty for that + row instead of reading them. + """ + + feasible: bool + growth: float = float("nan") + exch_fluxes: dict[str, float] = field(default_factory=dict) + block_fluxes: dict[str, float] = field(default_factory=dict) + + +def simulate_conditions( + model: "cobra.Model", + flux_data: "FluxData", + *, + constrain: bool, + zero_flux_rxns: list[str], + bio_rxn_id: str, + make_anaerobic: Optional[Callable[["cobra.Model"], None]] = None, + change_protein_biomass: Optional[Callable[["cobra.Model", float], None]] = None, + optimize: Optional[Callable[["cobra.Model"], "cobra.Solution"]] = None, +) -> list[ConditionSimResult]: + """Simulate every condition in one dataset. + + The steps are: + + 1. Every exchange reaction that appears as a *condition name* + anywhere in ``flux_data.conds`` (matched by name against + ``flux_data.exch_mets``) is blocked (``lower_bound = 0``) up + front, across the board -- not just the reaction for the row + being solved -- so one row's carbon source can't leak flux into + another row's simulation. + 2. For row ``i``, that row's own carbon-source reaction is + unblocked: fixed at the measured uptake rate if ``constrain`` + is True (``experiment_data.flux_data``), or fully opened to ``-1000`` + if ``constrain`` is False (``experiment_data.max_growth`` -- "what's + the *best possible* growth on this carbon source", not "growth + at this measured rate"). + 3. Optional per-condition adjustments run if the corresponding + callable is supplied: anaerobic switch when the oxygen exchange + column reads exactly 0, protein-content rescaling when + ``Ptot`` is given and nonzero. + 4. Solve; record growth, plus (``constrain`` datasets only, since + ``max_growth`` has no exchange measurements to compare against) + every non-NaN measured exchange's simulated flux and every + ``zero_flux_rxns`` entry's simulated flux. + + All mutation happens inside ``with model:``, which cobra reverts + on exit -- only bounds are touched here, so this is safe to call + repeatedly against one persistent (per-worker) model instance; a + copying the model per call is avoided (prohibitively + expensive on a real-scale model). + + Parameters + ---------- + model + The model, with every other constraint already applied. + flux_data + The dataset to simulate. + constrain + True for flux data (fix uptake at the measured rate), False + for max-growth data (open uptake fully). + zero_flux_rxns + Reaction IDs assumed zero-flux in every condition + (``experiment_data.zero_flux``); the row's own carbon source is + skipped even if it's also listed here. + bio_rxn_id + The biomass reaction ID. + make_anaerobic, change_protein_biomass + Optional organism-specific hooks, with no organism-agnostic default. + Data with no anaerobic or ``Ptot`` rows never needs them, and + passing ``None`` for either skips that adjustment instead of + raising. + optimize + Function that solves the model and returns a solution with + ``status``, ``objective_value`` and ``fluxes``. ``None`` calls + ``model.optimize()``. + + Returns + ------- + list[ConditionSimResult] + One entry per row of ``flux_data.conds``, in order. + + Raises + ------ + ValueError + If any ``flux_data.conds`` entry can't be matched by name to + an exchange metabolite in ``flux_data.exch_mets``. + """ + if optimize is None: + optimize = _default_optimize + cond_to_exch_idx = _match_conditions_to_exch(flux_data) + all_condition_rxn_ids = sorted( + {flux_data.exch_rxn_ids[j] for j in set(cond_to_exch_idx)} + ) + o2_col = _find_col(flux_data.exch_mets, "oxygen") + measured_cols = [ + j for j in range(len(flux_data.exch_rxn_ids)) + ] if constrain else [] + + results: list[ConditionSimResult] = [] + for i, exch_idx in enumerate(cond_to_exch_idx): + carbon_rxn_id = flux_data.exch_rxn_ids[exch_idx] + with model: + for rxn_id in all_condition_rxn_ids: + model.reactions.get_by_id(rxn_id).lower_bound = 0.0 + + carbon_rxn = model.reactions.get_by_id(carbon_rxn_id) + if constrain: + carbon_rxn.lower_bound = float(flux_data.exch_fluxes[i, exch_idx]) + else: + carbon_rxn.lower_bound = -1000.0 + + if o2_col is not None and flux_data.exch_fluxes[i, o2_col] == 0.0: + if make_anaerobic is not None: + make_anaerobic(model) + + ptot = float(flux_data.p_tot[i]) + if not (np.isnan(ptot) or ptot == 0.0): + if change_protein_biomass is not None: + change_protein_biomass(model, ptot) + + sol = optimize(model) + if ( + sol.status != "optimal" + or sol.objective_value is None + or np.isnan(sol.objective_value) + ): + results.append(ConditionSimResult(feasible=False)) + continue + + growth = float(sol.fluxes.get(bio_rxn_id, 0.0)) + exch_fluxes: dict[str, float] = {} + block_fluxes: dict[str, float] = {} + if constrain: + for j in measured_cols: + if np.isnan(flux_data.exch_fluxes[i, j]): + continue + rxn_id = flux_data.exch_rxn_ids[j] + exch_fluxes[rxn_id] = float(sol.fluxes.get(rxn_id, 0.0)) + for rxn_id in zero_flux_rxns: + if rxn_id == carbon_rxn_id: + continue + block_fluxes[rxn_id] = float(sol.fluxes.get(rxn_id, 0.0)) + results.append( + ConditionSimResult( + feasible=True, + growth=growth, + exch_fluxes=exch_fluxes, + block_fluxes=block_fluxes, + ) + ) + return results + + +def _default_optimize(model: "cobra.Model") -> "cobra.Solution": + return model.optimize() + + +def _match_conditions_to_exch(flux_data: "FluxData") -> list[int]: + """Column index into ``exch_mets``/``exch_rxn_ids`` for each + condition, matched by name. Raises ``ValueError`` when a condition + cannot be matched.""" + met_to_idx: dict[str, int] = {} + for j, met in enumerate(flux_data.exch_mets): + met_to_idx.setdefault(met, j) + missing = sorted({c for c in flux_data.conds if c not in met_to_idx}) + if missing: + raise ValueError( + 'Carbon source(s) "' + + "; ".join(missing) + + '" in the provided fluxData or maxGrowth cannot be matched ' + "by name with an exchange reaction." + ) + return [met_to_idx[c] for c in flux_data.conds] + + +def _find_col(exch_mets: list[str], name: str) -> Optional[int]: + try: + return exch_mets.index(name) + except ValueError: + return None diff --git a/src/raven_toolbox/experiments/constrain.py b/src/raven_toolbox/experiments/constrain.py new file mode 100644 index 0000000..1045bcf --- /dev/null +++ b/src/raven_toolbox/experiments/constrain.py @@ -0,0 +1,175 @@ +"""Set model exchange-flux bounds from one condition of a flux dataset. + +In a typical chemostat experiment you measure exchange fluxes +(substrate uptake, byproduct secretion, biomass growth rate) +under one or more growth conditions. :func:`apply_flux_data_constraints` takes a +:class:`~raven_toolbox.experiments.flux_data.FluxData`, selects one condition, +and applies those measurements as bounds on the matching +exchange reactions in the model. + +The ``loose_strict_flux`` argument controls how tightly: ``"loose"`` +gives the model room to wiggle around the measurement; a numeric +percentage clamps it within that band. +""" +from __future__ import annotations + +from typing import TYPE_CHECKING, Literal, Optional, Union + +import numpy as np + +if TYPE_CHECKING: + import cobra + + from .flux_data import FluxData + + +_LooseStrict = Union[Literal["loose"], float] + + +def apply_flux_data_constraints( + model: "cobra.Model", + flux_data: "FluxData", + *, + bio_rxn_id: str, + condition: int | str = 0, + max_min_growth: Literal["max", "min"] = "max", + loose_strict_flux: _LooseStrict = "loose", + c_source: Optional[str] = None, +) -> None: + """Constrain the model's exchange fluxes from a ``FluxData`` measurement. + + Sets the biomass reaction bounds to the measured growth rate and + each exchange reaction in ``flux_data.exch_rxn_ids`` to its + measured flux for the chosen condition. Also zeros out the + preferred carbon source so the data determines the choice. + + A ``+/-1000`` measured flux means unconstrained, regardless of + ``loose_strict_flux``: + ``-1000`` -> ``lb=-1000, ub=0`` (free uptake), ``+1000`` -> + ``lb=0, ub=1000`` (free excretion). + + Parameters + ---------- + model + Model with the exchange reactions referenced by + ``flux_data`` already in place. Mutated in place. + flux_data + Flux data, for example from + :func:`~raven_toolbox.experiments.flux_data.load_flux_data`. + bio_rxn_id + Biomass reaction ID. + condition + Either the 0-indexed condition number, or the condition name + (matched against ``flux_data.conds``). + max_min_growth + ``"max"`` sets ``bio_rxn_id`` upper bound to the measured + growth rate (and lower bound to 0); ``"min"`` sets the lower + bound (suitable when minimising a resource such as the protein + pool). + loose_strict_flux + ``"loose"`` keeps one bound at 0 and the other at the + measured value (allows zero flux). A numeric percentage + ``p`` brackets the measured value with hard bounds: + ``lb = val * (1 - p/200)``, ``ub = val * (1 + p/200)``, + swapped as needed to keep ``lb <= ub`` when ``val`` is + negative. ``p=10`` allows 10% total variance (i.e. +/-5%). + c_source + Reaction ID of the preferred carbon-source uptake reaction; + its bounds are set to ``(0, 0)`` before applying the flux + data so that the data alone determines the carbon source + used. ``None`` or ``""`` blocks no reaction. Silently ignored + if the reaction id is not found in the model. + + Raises + ------ + ValueError + If ``max_min_growth`` / ``loose_strict_flux`` is invalid, + ``condition`` (as string) is not in ``flux_data.conds``, or any + ``flux_data.exch_rxn_ids`` is missing from the model. + IndexError + If ``condition`` (as int) is out of range. + """ + if max_min_growth not in ("max", "min"): + raise ValueError( + f"max_min_growth must be 'max' or 'min', got {max_min_growth!r}" + ) + if not ( + loose_strict_flux == "loose" + or (isinstance(loose_strict_flux, (int, float)) + and not isinstance(loose_strict_flux, bool) + and 0 <= loose_strict_flux <= 100) + ): + raise ValueError( + f"loose_strict_flux must be 'loose' or a number in [0, 100]; " + f"got {loose_strict_flux!r}" + ) + + if isinstance(condition, str): + if condition not in flux_data.conds: + raise ValueError( + f"Condition {condition!r} not found in flux_data.conds " + f"({flux_data.conds})" + ) + cond_idx = flux_data.conds.index(condition) + else: + if not 0 <= condition < len(flux_data.conds): + raise IndexError( + f"condition index {condition} out of range " + f"(have {len(flux_data.conds)} condition(s))" + ) + cond_idx = condition + + fluxes = flux_data.exch_fluxes[cond_idx, :] + cobra_rxn_ids = {r.id for r in model.reactions} + missing = [r for r in flux_data.exch_rxn_ids if r not in cobra_rxn_ids] + if missing: + preview = missing[:5] + raise ValueError( + f"{len(missing)} exchange reaction ID(s) from flux_data are not " + f"present in the model (examples: {preview})" + ) + + if c_source: + try: + c_source_rxn = model.reactions.get_by_id(c_source) + c_source_rxn.lower_bound = 0.0 + c_source_rxn.upper_bound = 0.0 + except KeyError: + pass # c_source not in model; silent + + bio_rxn = model.reactions.get_by_id(bio_rxn_id) + gr = float(flux_data.gr_rate[cond_idx]) + if max_min_growth == "max": + bio_rxn.lower_bound = 0.0 + bio_rxn.upper_bound = gr + else: + bio_rxn.lower_bound = gr + bio_rxn.upper_bound = 1000.0 + + for rxn_id, flux in zip(flux_data.exch_rxn_ids, fluxes): + if np.isnan(flux): + continue + rxn = model.reactions.get_by_id(rxn_id) + + if abs(flux) == 1000.0: + if flux == -1000.0: + rxn.lower_bound = -1000.0 + rxn.upper_bound = 0.0 + else: + rxn.lower_bound = 0.0 + rxn.upper_bound = 1000.0 + continue + + if loose_strict_flux == "loose": + if flux < 0: + rxn.lower_bound = float(flux) + rxn.upper_bound = 0.0 + else: + rxn.lower_bound = 0.0 + rxn.upper_bound = float(flux) + else: + pct = float(loose_strict_flux) + lo = flux * (1 - pct / 200.0) + hi = flux * (1 + pct / 200.0) + rxn.lower_bound = float(min(lo, hi)) + rxn.upper_bound = float(max(lo, hi)) diff --git a/src/raven_toolbox/experiments/data.py b/src/raven_toolbox/experiments/data.py new file mode 100644 index 0000000..bca9578 --- /dev/null +++ b/src/raven_toolbox/experiments/data.py @@ -0,0 +1,88 @@ +"""Load the measured growth rates and fluxes of the conditions. + +The flux and max-growth files, ``fluxData.tsv`` and ``maxGrowth.tsv``, +share the column layout +``Condition Ptot grRate () ... weight source``, so both use +:func:`~raven_toolbox.experiments.flux_data.load_flux_data`; only ``zeroFlux.tsv`` +(a single ``Rxns`` id column) needs a loader of its own. A ``weight`` of 0 leaves a +condition out of the distance to the measurements. + +All three files are optional: a missing file yields ``None`` (or an empty +list for ``zero_flux``), never an exception. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +from .flux_data import FluxData, load_flux_data + +_FLUX_DATA_FILENAME = "fluxData.tsv" +_MAX_GROWTH_FILENAME = "maxGrowth.tsv" +_ZERO_EXCH_FILENAME = "zeroFlux.tsv" + + +@dataclass +class ExperimentData: + """Measured conditions. + + Attributes + ---------- + flux_data + Per-condition growth/exchange-flux measurements, from + ``fluxData.tsv``. ``None`` if that file isn't present. + max_growth + Per-condition maximum-growth-rate measurements (one active + carbon source per condition, at ``-1000`` i.e. unconstrained), + from ``maxGrowth.tsv``. ``None`` if that file isn't + present. + zero_flux + Reaction IDs assumed to carry zero flux, from + ``zeroFlux.tsv``. Empty if that file isn't present. + """ + + flux_data: Optional[FluxData] = None + max_growth: Optional[FluxData] = None + zero_flux: list[str] = field(default_factory=list) + + +def load_experiment_data(data_dir: str | Path) -> ExperimentData: + """Load the three experimental data files. + + Each file is optional and resolved as + ``/``; a missing file yields + ``None`` (``flux_data``/``max_growth``) or an empty list + (``zero_flux``) rather than raising. + + Parameters + ---------- + data_dir + The folder that holds the files. + + Returns + ------- + ExperimentData + """ + data_dir = Path(data_dir) + return ExperimentData( + flux_data=_load_optional_flux_data(data_dir / _FLUX_DATA_FILENAME), + max_growth=_load_optional_flux_data(data_dir / _MAX_GROWTH_FILENAME), + zero_flux=_load_zero_exch(data_dir / _ZERO_EXCH_FILENAME), + ) + + +def _load_optional_flux_data(path: Path) -> Optional[FluxData]: + if not path.is_file(): + return None + return load_flux_data(path) + + +def _load_zero_exch(path: Path) -> list[str]: + """Parse ``zeroFlux.tsv``: a single ``Rxns`` header, then + one reaction ID per line.""" + if not path.is_file(): + return [] + lines = path.read_text(encoding="utf-8").splitlines() + rxn_ids = [line.strip() for line in lines[1:] if line.strip()] + return rxn_ids diff --git a/src/raven_toolbox/experiments/distance.py b/src/raven_toolbox/experiments/distance.py new file mode 100644 index 0000000..5594471 --- /dev/null +++ b/src/raven_toolbox/experiments/distance.py @@ -0,0 +1,248 @@ +"""Carbon- and condition-weighted RMSE distance between a model and the data. + +:mod:`.conditions` does the FBA half; this module is pure numpy (plus one +small model-reading helper for the carbon weights) so the RMSE math is +directly testable against hand-computed expected values, with no model or +FBA involved. + +The biomass reaction is passed explicitly as ``bio_rxn_id`` +rather than matched by a literal name, and +:func:`dataset_rmse` builds its reaction-id-to-column lookup once. + +Every zero-carbon exchange metabolite (O2, water, ...) is clamped up to +a carbon weight of 1, not 0, so those reactions' flux mismatches still +count toward the RMSE at unit weight rather than being zeroed out of it. +See :func:`compute_carbon_weights`. +""" +from __future__ import annotations + +import warnings +from typing import TYPE_CHECKING, Iterable + +import numpy as np + +from .conditions import ConditionSimResult + +if TYPE_CHECKING: + import cobra + + from .data import ExperimentData + from .flux_data import FluxData + +#: Assumed carbon content of the biomass reaction, in Cmmol/gDCW, used to put growth-rate deviations on the +#: same carbon-weighted scale as exchange-flux deviations. +BIOMASS_CARBON_EQUIV = 41.0 + +#: Fixed penalty substituted for an infeasible/failed condition's RMSE +#: ("clearly worse than any real RMSE": it must dominate the mean). +INFEASIBLE_PENALTY = 99.0 + + +def compute_carbon_weights( + model: "cobra.Model", rxn_ids: Iterable[str], bio_rxn_id: str, +) -> dict[str, float]: + """Per-reaction carbon-weight lookup for RMSE weighting. + + The biomass reaction gets :data:`BIOMASS_CARBON_EQUIV`. Every + other reaction gets the carbon count of its single exchanged + metabolite (via cobrapy's parsed ``Metabolite.elements``, not a + hand-rolled formula parser). A missing/unparseable formula, or a + computed carbon count of exactly 0 (e.g. O2, water), both fall + back to 1. A model in which no + metabolite has a formula gives every exchange the weight 1 and + emits a warning. + + Parameters + ---------- + model + Model providing the metabolite formulas. + rxn_ids + Reaction IDs to compute weights for (typically every + reaction referenced by a dataset: its exchange + columns, its zero-flux list, and the biomass reaction). + bio_rxn_id + The biomass reaction ID. + + Returns + ------- + dict[str, float] + """ + carbon_weights: dict[str, float] = {} + if not any(m.formula for m in model.metabolites): + warnings.warn( + "The model has no metabolite formulas; every exchange " + "reaction gets the carbon weight 1.", + UserWarning, stacklevel=2, + ) + for rxn_id in rxn_ids: + if rxn_id == bio_rxn_id: + carbon_weights[rxn_id] = BIOMASS_CARBON_EQUIV + continue + rxn = model.reactions.get_by_id(rxn_id) + mets = list(rxn.metabolites) + carbon = mets[0].elements.get("C") if len(mets) == 1 else None + value = float(carbon) if carbon is not None else 1.0 + carbon_weights[rxn_id] = value if value != 0.0 else 1.0 + return carbon_weights + + +def dataset_rmse( + flux_data: "FluxData", + sims: list[ConditionSimResult], + *, + constrain: bool, + carbon_weights: dict[str, float], + bio_rxn_id: str, + penalty: float = INFEASIBLE_PENALTY, +) -> tuple[float, np.ndarray]: + """RMSE (mean, per-condition array) for one dataset. + + Parameters + ---------- + flux_data + The dataset being scored (``experiment_data.flux_data`` or + ``experiment_data.max_growth``). + sims + One :class:`~.conditions.ConditionSimResult` per + ``flux_data.conds`` row, e.g. from + :func:`~.conditions.simulate_conditions`. + constrain + Must match the value passed to + :func:`~.conditions.simulate_conditions` for these + ``sims``: True includes carbon-weighted exchange-flux terms + (flux data), False scores growth alone (max-growth data). + carbon_weights + Carbon-weight lookup from :func:`compute_carbon_weights`, covering + every reaction referenced by ``sims``' ``exch_fluxes`` / + ``block_fluxes`` plus ``bio_rxn_id``. A reaction missing from + the lookup falls back to a weight of 1. + bio_rxn_id + The biomass reaction ID (for reading ``flux_data.gr_rate``'s + carbon-equivalent counterpart). + penalty + RMSE substituted for an infeasible condition. + + Returns + ------- + rmse : float + Weighted (by ``flux_data.weight``, if present) + mean of the per-condition RMSEs. + rmse_list : numpy.ndarray + The per-condition RMSEs (post-penalty, pre-weighting), shape + ``(len(flux_data.conds),)``. + """ + n = len(flux_data.conds) + if len(sims) != n: + raise ValueError( + f"sims has {len(sims)} entries; expected {n} to match " + f"flux_data.conds." + ) + rxn_to_col = {r: j for j, r in enumerate(flux_data.exch_rxn_ids)} + + rmse_list = np.empty(n, dtype=float) + for i, sim in enumerate(sims): + if not sim.feasible: + rmse_list[i] = penalty + continue + + bio_meas = float(flux_data.gr_rate[i]) * BIOMASS_CARBON_EQUIV + bio_sim = sim.growth * BIOMASS_CARBON_EQUIV + + if not constrain: + rmse_list[i] = abs(bio_meas - bio_sim) + continue + + measured = [bio_meas] + simulated = [bio_sim] + for rxn_id, sim_flux in sim.exch_fluxes.items(): + w = carbon_weights.get(rxn_id, 1.0) + measured.append(w * float(flux_data.exch_fluxes[i, rxn_to_col[rxn_id]])) + simulated.append(w * sim_flux) + for rxn_id, sim_flux in sim.block_fluxes.items(): + w = carbon_weights.get(rxn_id, 1.0) + weighted_sim = w * sim_flux + if weighted_sim == 0.0: + # Already-correct zero predictions don't dilute the RMSE + continue + measured.append(0.0) + simulated.append(weighted_sim) + + diff = np.asarray(measured, dtype=float) - np.asarray(simulated, dtype=float) + rmse_list[i] = float(np.sqrt(np.mean(diff ** 2))) + + weighted = rmse_list + if flux_data.weight is not None: + weighted = rmse_list * flux_data.weight + return float(np.mean(weighted)), rmse_list + + +def experiment_distance( + experiment_data: "ExperimentData", + *, + flux_sims: list[ConditionSimResult] | None, + max_growth_sims: list[ConditionSimResult] | None, + carbon_weights: dict[str, float], + bio_rxn_id: str, + penalty: float = INFEASIBLE_PENALTY, + max_growth_weight: float = 1.0, +) -> tuple[float, dict[str, np.ndarray]]: + """Combine both datasets' RMSE into one score. + + ``flux_sims`` must be given (non-``None``) iff ``experiment_data.flux_data`` + is not ``None``, and likewise for ``max_growth_sims`` / + ``experiment_data.max_growth``. A dataset absent from ``experiment_data`` doesn't + contribute to the combined RMSE. + + Parameters + ---------- + max_growth_weight + Relative weight of the max-growth dataset against the flux + dataset, giving ``(rmse_flux + w * rmse_max_growth) / (w + 1)``. + At 2 the eight max-growth conditions have twice the weight of + the 33 flux conditions, which is the intended emphasis here: + the max-growth set is where the carbon-source diversity lives, + while 30 of the 33 flux conditions are glucose. + + + Returns + ------- + rmse : float + Weighted mean of whichever dataset RMSEs are present. ``NaN`` + if neither dataset is present. + detail : dict[str, numpy.ndarray] + Per-condition RMSE arrays, keyed ``"flux_data"`` / + ``"max_growth"`` for whichever datasets were scored. + """ + parts: list[float] = [] + weights: list[float] = [] + detail: dict[str, np.ndarray] = {} + + if experiment_data.flux_data is not None: + if flux_sims is None: + raise ValueError("experiment_data.flux_data is set but flux_sims is None.") + rmse, per_cond = dataset_rmse( + experiment_data.flux_data, flux_sims, + constrain=True, carbon_weights=carbon_weights, bio_rxn_id=bio_rxn_id, + penalty=penalty, + ) + parts.append(rmse) + weights.append(1.0) + detail["flux_data"] = per_cond + + if experiment_data.max_growth is not None: + if max_growth_sims is None: + raise ValueError("experiment_data.max_growth is set but max_growth_sims is None.") + rmse, per_cond = dataset_rmse( + experiment_data.max_growth, max_growth_sims, + constrain=False, carbon_weights=carbon_weights, bio_rxn_id=bio_rxn_id, + penalty=penalty, + ) + parts.append(rmse) + weights.append(float(max_growth_weight)) + detail["max_growth"] = per_cond + + if parts: + rmse = float(np.dot(parts, weights) / np.sum(weights)) + else: + rmse = float("nan") + return rmse, detail diff --git a/src/raven_toolbox/experiments/flux_data.py b/src/raven_toolbox/experiments/flux_data.py new file mode 100644 index 0000000..8231f25 --- /dev/null +++ b/src/raven_toolbox/experiments/flux_data.py @@ -0,0 +1,256 @@ +"""Container for the measured growth rates and exchange fluxes of several +conditions, and the parser of the ``fluxData.tsv`` format. +""" +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +import numpy as np + +logger = logging.getLogger(__name__) + + +# ` ()` -> capture met_name and rxn_id. + + +@dataclass +class FluxData: + """Per-condition flux measurements with per-reaction exchange data. + + Attributes + ---------- + conds + Names of the experimental conditions / samples. + p_tot + Total protein content per condition, g/gDCW. Shape + ``(n_conds,)``. + gr_rate + Measured growth rate per condition, 1/h. Shape ``(n_conds,)``. + exch_fluxes + Exchange flux per (condition, reaction), mmol/gDCW/h. Shape + ``(n_conds, n_rxns)``. NaN means "no measurement"; ``+/-1000`` + means unconstrained (used by + ``apply_flux_data_constraints``). + exch_mets + Metabolite names matching the columns of ``exch_fluxes``. + exch_rxn_ids + Exchange reaction IDs in the model, matching the columns of + ``exch_fluxes`` and ``exch_mets``. + weight + Optional per-condition weights of the + distance to the measurements (0 leaves the condition out). ``None`` if the source file did not have the + ``weight`` column. + source + Optional per-condition free-text describing where the data + came from. ``None`` if the source file did not have a + ``source`` column. + """ + + conds: list[str] = field(default_factory=list) + p_tot: np.ndarray = field( + default_factory=lambda: np.empty(0, dtype=float) + ) + gr_rate: np.ndarray = field( + default_factory=lambda: np.empty(0, dtype=float) + ) + exch_fluxes: np.ndarray = field( + default_factory=lambda: np.empty((0, 0), dtype=float) + ) + exch_mets: list[str] = field(default_factory=list) + exch_rxn_ids: list[str] = field(default_factory=list) + weight: Optional[np.ndarray] = None + source: Optional[list[str]] = None + + +def load_flux_data(path: str | Path) -> FluxData: + """Parse a ``fluxData.tsv`` into a ``FluxData``. + + File format (tab-delimited, one header row): + + condition Ptot grRate () () ... + c1 0.5 0.4 -10.0 5.0 ... + c2 0.6 0.5 -8.5 4.2 ... + + Optional extra columns (in any position): + + * ``weight`` (float): weight of the condition in the distance to the measurements. + * ``source`` (str): free-text describing the data origin. + + An exchange column header has the form `` ()``, split at + the parenthesis that matches the final closing one, so a reaction id + may contain parentheses (``X (EX_a(e))`` gives ``X`` and + ``EX_a(e)``). + + Both columns are dropped from the matrix and surfaced on the + returned ``FluxData`` as ``weight`` and ``source``. + + Parameters + ---------- + path + Path to the TSV file. + + Returns + ------- + FluxData + + Raises + ------ + FileNotFoundError + If ``path`` does not exist. + ValueError + If the file has no header row, no exchange columns, or a row + with a different number of cells than the header. + """ + path = Path(path) + if not path.is_file(): + raise FileNotFoundError(f"fluxData file not found: {path}") + + rows: list[list[str]] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + rows.append(line.rstrip("\n").split("\t")) + + if not rows: + raise ValueError(f"{path} is empty.") + if len(rows[0]) < 3: + raise ValueError( + f"{path}: header has only {len(rows[0])} column(s); " + f"expected at least 'condition Ptot grRate'." + ) + + header = rows[0] + data = rows[1:] + for line_no, r in enumerate(data, start=2): + if len(r) != len(header): + raise ValueError( + f"{path}: row {line_no} has {len(r)} cell(s), the header " + f"has {len(header)}." + ) + + # Pull out optional columns first; collect indices to drop. + optional_indices: list[int] = [] + if _index_of(header, "evotuneRMSEweight") is not None: + raise ValueError( + f"{path}: the column 'evotuneRMSEweight' is now called 'weight'." + ) + weight_idx = _index_of(header, "weight") + source_idx = _index_of(header, "source") + + weights: Optional[np.ndarray] = None + source_values: Optional[list[str]] = None + + if weight_idx is not None: + weights = np.array( + [_parse_float(r[weight_idx]) for r in data], + dtype=float, + ) + optional_indices.append(weight_idx) + if source_idx is not None: + source_values = [r[source_idx] for r in data] + optional_indices.append(source_idx) + + # Build the kept-column index list (preserving order). + kept = [j for j in range(len(header)) if j not in optional_indices] + kept_header = [header[j] for j in kept] + + # First three columns are condition / Ptot / grRate. Remaining are exchanges. + if len(kept_header) < 4: + raise ValueError( + f"{path}: no exchange-flux columns found after the first three." + ) + + exch_headers = kept_header[3:] + exch_mets, exch_rxn_ids = _split_exch_headers(exch_headers) + + conds = [r[kept[0]] for r in data] + p_tot = np.array([_parse_float(r[kept[1]]) for r in data], dtype=float) + gr_rate = np.array([_parse_float(r[kept[2]]) for r in data], dtype=float) + exch_fluxes = np.full((len(data), len(exch_headers)), np.nan, dtype=float) + for i, r in enumerate(data): + for ci, j in enumerate(kept[3:]): + exch_fluxes[i, ci] = _parse_float(r[j]) + + return FluxData( + conds=conds, + p_tot=p_tot, + gr_rate=gr_rate, + exch_fluxes=exch_fluxes, + exch_mets=exch_mets, + exch_rxn_ids=exch_rxn_ids, + weight=weights, + source=source_values, + ) + + +def _index_of(header: list[str], name: str) -> Optional[int]: + """Return the index of `name` in `header`, or None if absent.""" + for j, h in enumerate(header): + if h.strip() == name: + return j + return None + + +def _parse_float(s: str) -> float: + """Parse a numeric cell; empty / non-numeric -> NaN.""" + s = s.strip() + if not s: + return float("nan") + try: + return float(s) + except ValueError: + return float("nan") + + +def _split_header(header: str) -> Optional[tuple[str, str]]: + """Split ``' ()'`` at the parenthesis that matches the final + closing one; return None for a header of another form.""" + if not header.endswith(")"): + return None + depth = 0 + for k in range(len(header) - 1, -1, -1): + if header[k] == ")": + depth += 1 + elif header[k] == "(": + depth -= 1 + if depth == 0: + rxn = header[k + 1:-1].strip() + if not rxn: + return None + return header[:k].strip(), rxn + return None + + +def _split_exch_headers(headers: list[str]) -> tuple[list[str], list[str]]: + """Split each header `' ()'` into (met, rxn) pairs. + + Headers without parentheses pass through with the rxn_id equal + to the full header (the + capture-group falls through unchanged). + """ + mets: list[str] = [] + rxns: list[str] = [] + no_paren: list[str] = [] + for h in headers: + parts = _split_header(h.strip()) + if parts is not None: + mets.append(parts[0]) + rxns.append(parts[1]) + else: + mets.append(h.strip()) + rxns.append(h.strip()) + no_paren.append(h.strip()) + if no_paren: + # The reaction id is taken from the header, so a column without the + # ` ()` form will be matched against a reaction whose id is + # the metabolite name -- almost certainly wrong. Surface it. + logger.warning( + "load_flux_data: %d exchange column header(s) lack the " + "' ()' form; their reaction id falls back to the full " + "header text: %s", + len(no_paren), no_paren[:5], + ) + return mets, rxns diff --git a/tests/test_experiments_conditions.py b/tests/test_experiments_conditions.py new file mode 100644 index 0000000..89e0397 --- /dev/null +++ b/tests/test_experiments_conditions.py @@ -0,0 +1,476 @@ +"""Tests for experiments.conditions and experiments.distance. + +A tiny two-carbon-source toy model with a known enzyme cap, so every +simulated growth/exchange-flux value used below is hand-predictable +from the model's stoichiometry rather than treated as a black box. +""" +import cobra +import numpy as np +import pytest + +from raven_toolbox.experiments.data import ExperimentData +from raven_toolbox.experiments.flux_data import FluxData +from raven_toolbox.experiments.distance import ( + BIOMASS_CARBON_EQUIV, + experiment_distance, + compute_carbon_weights, + dataset_rmse, +) +from raven_toolbox.experiments.conditions import ( + ConditionSimResult, + simulate_conditions, +) + + +# --------------------------------------------------------------------------- # +# Toy two-carbon-source model fixture +# --------------------------------------------------------------------------- # +# +# glucose and ethanol each feed a shared pool metabolite A_c, which the +# single enzyme-gated reaction R turns into biomass at an enzyme-capped +# rate of kcat*3600/mw = 1*3600/100 = 36 (pool size 1 mg/gDW). R also +# co-produces a small fixed amount of a "byproduct" the experimental +# data assumes is never made (byp_c, 0.1 mol per mol of R flux) -- this +# exercises `zero_flux_rxns`' nonzero-simulated-flux case. +# +# - "glucose" condition: uptake unconstrained relative to the enzyme +# cap (measured flux -36 exactly matches TR's capacity), so growth +# is enzyme-limited: 36. +# - "ethanol" condition: uptake capped at -20 (below the enzyme cap), +# so growth is carbon-limited instead: 20. + +_KCAT = 1.0 +_MW = 100.0 +_ENZYME_CAP = _KCAT * 3600.0 / _MW # 36.0 +_BYPRODUCT_YIELD = 0.1 + + +def _build_toy() -> cobra.Model: + model = cobra.Model("toy") + + glc_e = cobra.Metabolite("glc_e", compartment="e", formula="C6H12O6") + eth_e = cobra.Metabolite("eth_e", compartment="e", formula="C2H6O") + byp_c = cobra.Metabolite("byp_c", compartment="c", formula="C1") + A_c = cobra.Metabolite("A_c", compartment="c") + pool = cobra.Metabolite("prot_pool", compartment="c") + prot_E = cobra.Metabolite("prot_E", compartment="c") + bio_met = cobra.Metabolite("bio_met", compartment="c") + model.add_metabolites([glc_e, eth_e, byp_c, A_c, pool, prot_E, bio_met]) + + EX_glc = cobra.Reaction("EX_glc") + EX_glc.add_metabolites({glc_e: -1.0}) + EX_glc.bounds = (-1000.0, 0.0) + + EX_eth = cobra.Reaction("EX_eth") + EX_eth.add_metabolites({eth_e: -1.0}) + EX_eth.bounds = (-1000.0, 0.0) + + EX_byp = cobra.Reaction("EX_byp") + EX_byp.add_metabolites({byp_c: -1.0}) + EX_byp.bounds = (0.0, 1000.0) + + TR_glc = cobra.Reaction("TR_glc") + TR_glc.add_metabolites({glc_e: -1.0, A_c: 1.0}) + TR_glc.bounds = (0.0, 1000.0) + + TR_eth = cobra.Reaction("TR_eth") + TR_eth.add_metabolites({eth_e: -1.0, A_c: 1.0}) + TR_eth.bounds = (0.0, 1000.0) + + coeff = _MW / (_KCAT * 3600.0) # mg/mmol per unit flux + R = cobra.Reaction("R") + R.add_metabolites( + {A_c: -1.0, prot_E: -coeff, bio_met: 1.0, byp_c: _BYPRODUCT_YIELD} + ) + R.bounds = (0.0, 1000.0) + + BIO = cobra.Reaction("biomass") + BIO.add_metabolites({bio_met: -1.0}) + BIO.bounds = (0.0, 1000.0) + + pool_ex = cobra.Reaction("prot_pool_exchange") + pool_ex.add_metabolites({pool: 1.0}) + pool_ex.bounds = (0.0, 1.0) # 1 mg/gDW pool + + usage_E = cobra.Reaction("usage_prot_E") + usage_E.add_metabolites({pool: -1.0, prot_E: 1.0}) + usage_E.bounds = (0.0, 1000.0) + + model.add_reactions( + [EX_glc, EX_eth, EX_byp, TR_glc, TR_eth, R, BIO, pool_ex, usage_E] + ) + model.objective = "biomass" + return model + + +def _flux_data(*, ethanol_grrate: float) -> FluxData: + """Two conditions: glucose (enzyme-limited, everything matches + exactly) and ethanol (carbon-limited; ``ethanol_grrate`` controls + whether the measured growth matches the simulated 20.0 exactly or + deliberately deviates from it).""" + return FluxData( + conds=["glucose", "ethanol"], + p_tot=np.array([np.nan, np.nan]), + gr_rate=np.array([_ENZYME_CAP, ethanol_grrate]), + exch_fluxes=np.array( + [ + [-_ENZYME_CAP, np.nan], + [np.nan, -20.0], + ] + ), + exch_mets=["glucose", "ethanol"], + exch_rxn_ids=["EX_glc", "EX_eth"], + weight=np.array([1.0, 1.0]), + source=["test", "test"], + ) + + +def _max_growth_data() -> FluxData: + """Max-growth dataset: uptake fully opened, so both conditions hit + the same enzyme-capped growth of 36.""" + return FluxData( + conds=["glucose", "ethanol"], + p_tot=np.array([np.nan, np.nan]), + gr_rate=np.array([_ENZYME_CAP, _ENZYME_CAP]), + exch_fluxes=np.array( + [ + [-1000.0, np.nan], + [np.nan, -1000.0], + ] + ), + exch_mets=["glucose", "ethanol"], + exch_rxn_ids=["EX_glc", "EX_eth"], + ) + + +# --------------------------------------------------------------------------- # +# simulate_conditions +# --------------------------------------------------------------------------- # + +def test_simulate_flux_data_enzyme_vs_carbon_limited(): + model = _build_toy() + flux_data = _flux_data(ethanol_grrate=20.0) + + results = simulate_conditions( + model, flux_data, + constrain=True, zero_flux_rxns=["EX_byp"], bio_rxn_id="biomass", + ) + + assert len(results) == 2 + glucose, ethanol = results + + # Glucose: enzyme-limited growth of 36, full carbon uptake used. + assert glucose.feasible + assert glucose.growth == pytest.approx(_ENZYME_CAP) + assert glucose.exch_fluxes["EX_glc"] == pytest.approx(-_ENZYME_CAP) + assert glucose.block_fluxes["EX_byp"] == pytest.approx( + _ENZYME_CAP * _BYPRODUCT_YIELD + ) + + # Ethanol: carbon-limited to the -20 uptake cap, growth = 20. + assert ethanol.feasible + assert ethanol.growth == pytest.approx(20.0) + assert ethanol.exch_fluxes["EX_eth"] == pytest.approx(-20.0) + assert ethanol.block_fluxes["EX_byp"] == pytest.approx(20.0 * _BYPRODUCT_YIELD) + + # Bounds fully revert after the call (both `with model:` blocks exited). + assert model.reactions.get_by_id("EX_glc").bounds == (-1000.0, 0.0) + assert model.reactions.get_by_id("EX_eth").bounds == (-1000.0, 0.0) + + +def test_simulate_max_growth_opens_uptake_fully(): + model = _build_toy() + max_growth = _max_growth_data() + + results = simulate_conditions( + model, max_growth, + constrain=False, zero_flux_rxns=[], bio_rxn_id="biomass", + ) + + assert [r.growth for r in results] == pytest.approx([_ENZYME_CAP, _ENZYME_CAP]) + # max_growth rows carry no exchange-flux measurements to report back. + assert results[0].exch_fluxes == {} + assert results[0].block_fluxes == {} + + +def test_simulate_unmatched_condition_name_raises(): + model = _build_toy() + flux_data = _flux_data(ethanol_grrate=20.0) + flux_data.conds = ["glucose", "propionate"] + + with pytest.raises(ValueError, match="propionate"): + simulate_conditions( + model, flux_data, + constrain=True, zero_flux_rxns=[], bio_rxn_id="biomass", + ) + + +# --------------------------------------------------------------------------- # +# compute_carbon_weights +# --------------------------------------------------------------------------- # + +def test_compute_carbon_weights(): + model = _build_toy() + + carbon_weights = compute_carbon_weights( + model, ["EX_glc", "EX_eth", "EX_byp", "biomass"], bio_rxn_id="biomass", + ) + + assert carbon_weights["EX_glc"] == pytest.approx(6.0) # C6H12O6 + assert carbon_weights["EX_eth"] == pytest.approx(2.0) # C2H6O + assert carbon_weights["EX_byp"] == pytest.approx(1.0) # C1 + assert carbon_weights["biomass"] == pytest.approx(BIOMASS_CARBON_EQUIV) + + +def test_compute_carbon_weights_zero_carbon_clamps_to_one(): + """A genuinely zero-carbon exchanged metabolite (e.g. O2) still + gets weight 1, not 0, so those reactions' mismatches are not silently + excluded from the RMSE.""" + model = _build_toy() + o2_e = cobra.Metabolite("o2_e", compartment="e", formula="O2") + model.add_metabolites([o2_e]) + EX_o2 = cobra.Reaction("EX_o2") + EX_o2.add_metabolites({o2_e: -1.0}) + EX_o2.bounds = (-1000.0, 0.0) + model.add_reactions([EX_o2]) + + carbon_weights = compute_carbon_weights(model, ["EX_o2"], bio_rxn_id="biomass") + + assert carbon_weights["EX_o2"] == pytest.approx(1.0) + + +# --------------------------------------------------------------------------- # +# dataset_rmse / experiment_distance +# --------------------------------------------------------------------------- # + +def test_dataset_rmse_measured_terms_match_only_byproduct_deviates(): + model = _build_toy() + # ethanol_grrate=20.0 matches the ethanol condition's actual + # carbon-limited growth exactly, and both conditions' own measured + # exchange fluxes match the simulation exactly too -- the only + # nonzero term left is the unmeasured byproduct (assumed + # zero-flux, but nonzero in the simulation). + flux_data = _flux_data(ethanol_grrate=20.0) + sims = simulate_conditions( + model, flux_data, + constrain=True, zero_flux_rxns=["EX_byp"], bio_rxn_id="biomass", + ) + carbon_weights = compute_carbon_weights( + model, ["EX_glc", "EX_eth", "EX_byp", "biomass"], bio_rxn_id="biomass", + ) + + rmse, per_cond = dataset_rmse( + flux_data, sims, + constrain=True, carbon_weights=carbon_weights, bio_rxn_id="biomass", + ) + + # The byproduct term is nonzero-simulated-vs-zero-measured, so RMSE + # is not zero overall -- but every *measured* exchange/growth term + # matches exactly, and the byproduct term is hand-computable. + glucose_byp = _ENZYME_CAP * _BYPRODUCT_YIELD * 1.0 # carbon weight 1 + ethanol_byp = 20.0 * _BYPRODUCT_YIELD * 1.0 + # 3 terms per condition: growth (matches exactly -> 0), the + # condition's own exchange flux (matches exactly -> 0), and the + # byproduct block term (measured 0 vs the nonzero simulated value). + expected_glucose_rmse = np.sqrt(np.mean([0.0, 0.0, glucose_byp ** 2])) + expected_ethanol_rmse = np.sqrt(np.mean([0.0, 0.0, ethanol_byp ** 2])) + + assert per_cond[0] == pytest.approx(expected_glucose_rmse) + assert per_cond[1] == pytest.approx(expected_ethanol_rmse) + assert rmse == pytest.approx(np.mean([expected_glucose_rmse, expected_ethanol_rmse])) + + +def test_dataset_rmse_growth_mismatch_is_hand_computable(): + model = _build_toy() + # Deliberately wrong measured growth for the ethanol condition + # (18.0 vs the actual carbon-limited 20.0) -- a known, hand-provable + # deviation on top of the byproduct term. + flux_data = _flux_data(ethanol_grrate=18.0) + sims = simulate_conditions( + model, flux_data, + constrain=True, zero_flux_rxns=["EX_byp"], bio_rxn_id="biomass", + ) + carbon_weights = compute_carbon_weights( + model, ["EX_glc", "EX_eth", "EX_byp", "biomass"], bio_rxn_id="biomass", + ) + + _, per_cond = dataset_rmse( + flux_data, sims, + constrain=True, carbon_weights=carbon_weights, bio_rxn_id="biomass", + ) + + growth_diff = (18.0 - 20.0) * BIOMASS_CARBON_EQUIV # bio_meas - bio_sim + byp_diff = 0.0 - (20.0 * _BYPRODUCT_YIELD) # measured 0 vs simulated + expected = np.sqrt(np.mean([growth_diff ** 2, 0.0, byp_diff ** 2])) + assert per_cond[1] == pytest.approx(expected) + + +def test_dataset_rmse_infeasible_condition_gets_penalty(): + flux_data = _flux_data(ethanol_grrate=20.0) + sims = [ConditionSimResult(feasible=False), ConditionSimResult(feasible=False)] + carbon_weights = {} + + rmse, per_cond = dataset_rmse( + flux_data, sims, + constrain=True, carbon_weights=carbon_weights, bio_rxn_id="biomass", penalty=99.0, + ) + + assert list(per_cond) == [99.0, 99.0] + assert rmse == pytest.approx(99.0) + + +def test_dataset_rmse_max_growth_scores_growth_only(): + max_growth = _max_growth_data() + sims = [ + ConditionSimResult(feasible=True, growth=_ENZYME_CAP), + ConditionSimResult(feasible=True, growth=30.0), # deliberately off + ] + + rmse, per_cond = dataset_rmse( + max_growth, sims, + constrain=False, carbon_weights={}, bio_rxn_id="biomass", + ) + + assert per_cond[0] == pytest.approx(0.0) + expected_second = abs((_ENZYME_CAP - 30.0) * BIOMASS_CARBON_EQUIV) + assert per_cond[1] == pytest.approx(expected_second) + assert rmse == pytest.approx(np.mean([0.0, expected_second])) + + +def test_experiment_distance_combines_both_datasets(): + model = _build_toy() + flux_data = _flux_data(ethanol_grrate=20.0) + max_growth = _max_growth_data() + experiment_data = ExperimentData(flux_data=flux_data, max_growth=max_growth, zero_flux=["EX_byp"]) + + flux_sims = simulate_conditions( + model, flux_data, + constrain=True, zero_flux_rxns=["EX_byp"], bio_rxn_id="biomass", + ) + max_growth_sims = simulate_conditions( + model, max_growth, + constrain=False, zero_flux_rxns=[], bio_rxn_id="biomass", + ) + carbon_weights = compute_carbon_weights( + model, ["EX_glc", "EX_eth", "EX_byp", "biomass"], bio_rxn_id="biomass", + ) + + rmse, detail = experiment_distance( + experiment_data, + flux_sims=flux_sims, max_growth_sims=max_growth_sims, + carbon_weights=carbon_weights, bio_rxn_id="biomass", + ) + + expected_flux_rmse, _ = dataset_rmse( + flux_data, flux_sims, constrain=True, carbon_weights=carbon_weights, bio_rxn_id="biomass", + ) + expected_max_growth_rmse, _ = dataset_rmse( + max_growth, max_growth_sims, constrain=False, carbon_weights=carbon_weights, + bio_rxn_id="biomass", + ) + assert rmse == pytest.approx( + np.mean([expected_flux_rmse, expected_max_growth_rmse]) + ) + assert "flux_data" in detail and "max_growth" in detail + + +def test_experiment_distance_max_growth_weight_scales_the_max_growth_term(): + """``max_growth_weight`` scales the max-growth term, so 2 makes the + eight max-growth conditions count double against the 33 flux ones.""" + model = _build_toy() + flux_data = _flux_data(ethanol_grrate=20.0) + max_growth = _max_growth_data() + experiment_data = ExperimentData(flux_data=flux_data, max_growth=max_growth, zero_flux=["EX_byp"]) + + flux_sims = simulate_conditions( + model, flux_data, + constrain=True, zero_flux_rxns=["EX_byp"], bio_rxn_id="biomass", + ) + max_growth_sims = simulate_conditions( + model, max_growth, + constrain=False, zero_flux_rxns=[], bio_rxn_id="biomass", + ) + carbon_weights = compute_carbon_weights( + model, ["EX_glc", "EX_eth", "EX_byp", "biomass"], bio_rxn_id="biomass", + ) + + expected_flux_rmse, _ = dataset_rmse( + flux_data, flux_sims, constrain=True, carbon_weights=carbon_weights, bio_rxn_id="biomass", + ) + expected_max_growth_rmse, _ = dataset_rmse( + max_growth, max_growth_sims, constrain=False, carbon_weights=carbon_weights, + bio_rxn_id="biomass", + ) + + rmse, _ = experiment_distance( + experiment_data, + flux_sims=flux_sims, max_growth_sims=max_growth_sims, + carbon_weights=carbon_weights, bio_rxn_id="biomass", max_growth_weight=2.0, + ) + assert rmse == pytest.approx( + (expected_flux_rmse + 2.0 * expected_max_growth_rmse) / 3.0 + ) + + # 0.5 weights the flux term twice as much as the max-growth term. + rmse_half, _ = experiment_distance( + experiment_data, + flux_sims=flux_sims, max_growth_sims=max_growth_sims, + carbon_weights=carbon_weights, bio_rxn_id="biomass", max_growth_weight=0.5, + ) + assert rmse_half == pytest.approx( + (2.0 * expected_flux_rmse + expected_max_growth_rmse) / 3.0 + ) + + # A lone dataset carries the whole score whatever the weight. + solo = ExperimentData(flux_data=flux_data, max_growth=None, zero_flux=["EX_byp"]) + rmse_solo, _ = experiment_distance( + solo, + flux_sims=flux_sims, max_growth_sims=None, + carbon_weights=carbon_weights, bio_rxn_id="biomass", max_growth_weight=2.0, + ) + assert rmse_solo == pytest.approx(expected_flux_rmse) + + +def test_experiment_distance_missing_dataset_contributes_nothing(): + model = _build_toy() + flux_data = _flux_data(ethanol_grrate=20.0) + experiment_data = ExperimentData(flux_data=flux_data, max_growth=None, zero_flux=["EX_byp"]) + + flux_sims = simulate_conditions( + model, flux_data, + constrain=True, zero_flux_rxns=["EX_byp"], bio_rxn_id="biomass", + ) + carbon_weights = compute_carbon_weights( + model, ["EX_glc", "EX_eth", "EX_byp", "biomass"], bio_rxn_id="biomass", + ) + + rmse, detail = experiment_distance( + experiment_data, + flux_sims=flux_sims, max_growth_sims=None, + carbon_weights=carbon_weights, bio_rxn_id="biomass", + ) + + expected_flux_rmse, _ = dataset_rmse( + flux_data, flux_sims, constrain=True, carbon_weights=carbon_weights, bio_rxn_id="biomass", + ) + assert rmse == pytest.approx(expected_flux_rmse) + assert set(detail) == {"flux_data"} + + +def test_simulate_uses_the_given_optimize_function(): + """``optimize`` replaces ``model.optimize()`` for every condition.""" + model = _build_toy() + calls = [] + + def optimize(m): + calls.append(m) + return m.optimize() + + sims = simulate_conditions( + model, _flux_data(ethanol_grrate=20.0), + constrain=True, zero_flux_rxns=[], bio_rxn_id="biomass", + optimize=optimize, + ) + + assert len(calls) == 2 + assert all(s.feasible for s in sims) diff --git a/tests/test_experiments_constrain.py b/tests/test_experiments_constrain.py new file mode 100644 index 0000000..72d5541 --- /dev/null +++ b/tests/test_experiments_constrain.py @@ -0,0 +1,329 @@ +"""Tests for apply_flux_data_constraints.""" +import cobra +import numpy as np +import pytest + +from raven_toolbox.experiments import FluxData, apply_flux_data_constraints + + +# --------------------------------------------------------------------------- # +# Fixture builders +# --------------------------------------------------------------------------- # + +def _build_simple_model() -> cobra.Model: + """Tiny model with biomass + a few exchange rxns.""" + model = cobra.Model("toy") + + glc = cobra.Metabolite("glc_e", compartment="e") + O2 = cobra.Metabolite("O2_e", compartment="e") + eth = cobra.Metabolite("eth_e", compartment="e") + bio_met = cobra.Metabolite("bio", compartment="c") + model.add_metabolites([glc, O2, eth, bio_met]) + + EX_glc = cobra.Reaction("EX_glc") + EX_glc.add_metabolites({glc: -1.0}) + EX_glc.lower_bound = -1000.0 + EX_glc.upper_bound = 1000.0 + + EX_O2 = cobra.Reaction("EX_O2") + EX_O2.add_metabolites({O2: -1.0}) + EX_O2.lower_bound = -1000.0 + EX_O2.upper_bound = 1000.0 + + EX_eth = cobra.Reaction("EX_eth") + EX_eth.add_metabolites({eth: -1.0}) + EX_eth.lower_bound = -1000.0 + EX_eth.upper_bound = 1000.0 + + BIO = cobra.Reaction("biomass") + BIO.add_metabolites({bio_met: 1.0}) + BIO.lower_bound = 0.0 + BIO.upper_bound = 1000.0 + + model.add_reactions([EX_glc, EX_O2, EX_eth, BIO]) + + return model + + +def _flux_data( + *, + conds: list[str], + p_tot: list[float], + gr_rate: list[float], + exch_fluxes: list[list[float]], + exch_rxn_ids: list[str], +) -> FluxData: + return FluxData( + conds=list(conds), + p_tot=np.array(p_tot, dtype=float), + gr_rate=np.array(gr_rate, dtype=float), + exch_fluxes=np.array(exch_fluxes, dtype=float), + exch_mets=list(exch_rxn_ids), + exch_rxn_ids=list(exch_rxn_ids), + ) + + +# --------------------------------------------------------------------------- # +# Pre-flight +# --------------------------------------------------------------------------- # + +def test_invalid_max_min_growth_raises(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-5.0]], exch_rxn_ids=["EX_glc"], + ) + with pytest.raises(ValueError, match="max_min_growth"): + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", max_min_growth="invalid") + + +def test_invalid_loose_strict_raises(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-5.0]], exch_rxn_ids=["EX_glc"], + ) + with pytest.raises(ValueError, match="loose_strict_flux"): + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", loose_strict_flux=200) + + +def test_unknown_condition_name_raises(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-5.0]], exch_rxn_ids=["EX_glc"], + ) + with pytest.raises(ValueError, match="Condition"): + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", condition="nonexistent") + + +def test_out_of_range_condition_index_raises(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-5.0]], exch_rxn_ids=["EX_glc"], + ) + with pytest.raises(IndexError, match="condition index"): + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", condition=5) + + +def test_unknown_rxn_id_raises(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-5.0]], exch_rxn_ids=["EX_nonexistent"], + ) + with pytest.raises(ValueError, match="not present in the model"): + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass") + + +# --------------------------------------------------------------------------- # +# Growth bounds +# --------------------------------------------------------------------------- # + +def test_max_growth_sets_upper_bound(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-5.0]], exch_rxn_ids=["EX_glc"], + ) + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", max_min_growth="max") + bio = model.reactions.get_by_id("biomass") + assert bio.lower_bound == 0.0 + assert bio.upper_bound == pytest.approx(0.4) + + +def test_min_growth_sets_lower_bound(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-5.0]], exch_rxn_ids=["EX_glc"], + ) + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", max_min_growth="min") + bio = model.reactions.get_by_id("biomass") + assert bio.lower_bound == pytest.approx(0.4) + assert bio.upper_bound == 1000.0 + + +# --------------------------------------------------------------------------- # +# c_source reset +# --------------------------------------------------------------------------- # + +def test_c_source_zeroed(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-5.0]], exch_rxn_ids=["EX_glc"], + ) + apply_flux_data_constraints( + model, fd, bio_rxn_id="biomass", c_source="EX_eth", + ) + eth_rxn = model.reactions.get_by_id("EX_eth") + assert eth_rxn.lower_bound == 0.0 + assert eth_rxn.upper_bound == 0.0 + + +# --------------------------------------------------------------------------- # +# Loose mode +# --------------------------------------------------------------------------- # + +def test_loose_negative_flux_caps_lb(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-5.0]], exch_rxn_ids=["EX_glc"], + ) + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", loose_strict_flux="loose") + glc_rxn = model.reactions.get_by_id("EX_glc") + assert glc_rxn.lower_bound == -5.0 + assert glc_rxn.upper_bound == 0.0 + + +def test_loose_positive_flux_caps_ub(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[3.0]], exch_rxn_ids=["EX_eth"], + ) + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", loose_strict_flux="loose") + eth_rxn = model.reactions.get_by_id("EX_eth") + assert eth_rxn.lower_bound == 0.0 + assert eth_rxn.upper_bound == 3.0 + + +def test_loose_nan_flux_skipped(): + model = _build_simple_model() + original_lb = model.reactions.get_by_id("EX_O2").lower_bound + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[np.nan]], exch_rxn_ids=["EX_O2"], + ) + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass") + # NaN means "no measurement"; bounds untouched. + assert model.reactions.get_by_id("EX_O2").lower_bound == original_lb + + +# --------------------------------------------------------------------------- # +# Percentage variance mode +# --------------------------------------------------------------------------- # + +def test_percentage_brackets_positive_value(): + """pct=10 -> lb = val*0.95, ub = val*1.05.""" + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[3.0]], exch_rxn_ids=["EX_eth"], + ) + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", loose_strict_flux=10) + eth_rxn = model.reactions.get_by_id("EX_eth") + assert eth_rxn.lower_bound == pytest.approx(2.85) + assert eth_rxn.upper_bound == pytest.approx(3.15) + + +def test_percentage_handles_negative_value(): + """For negative val, lb and ub are swapped to keep lb <= ub.""" + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-5.0]], exch_rxn_ids=["EX_glc"], + ) + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", loose_strict_flux=10) + glc_rxn = model.reactions.get_by_id("EX_glc") + # val=-5, pct=10 -> raw [-5*0.95, -5*1.05] = [-4.75, -5.25] + # After swap: lb = -5.25, ub = -4.75. + assert glc_rxn.lower_bound == pytest.approx(-5.25) + assert glc_rxn.upper_bound == pytest.approx(-4.75) + + +# --------------------------------------------------------------------------- # +# Extreme +/-1000 sentinel +# --------------------------------------------------------------------------- # + +def test_minus_thousand_is_unconstrained_uptake(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-1000.0]], exch_rxn_ids=["EX_O2"], + ) + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", loose_strict_flux=10) + o2_rxn = model.reactions.get_by_id("EX_O2") + assert o2_rxn.lower_bound == -1000.0 + assert o2_rxn.upper_bound == 0.0 + + +def test_plus_thousand_is_unconstrained_excretion(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[1000.0]], exch_rxn_ids=["EX_eth"], + ) + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", loose_strict_flux=10) + eth_rxn = model.reactions.get_by_id("EX_eth") + assert eth_rxn.lower_bound == 0.0 + assert eth_rxn.upper_bound == 1000.0 + + +# --------------------------------------------------------------------------- # +# Condition selection +# --------------------------------------------------------------------------- # + +def test_condition_by_index(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1", "c2"], + p_tot=[0.5, 0.6], + gr_rate=[0.4, 0.7], + exch_fluxes=[[-5.0], [-8.0]], + exch_rxn_ids=["EX_glc"], + ) + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", condition=1) + bio = model.reactions.get_by_id("biomass") + assert bio.upper_bound == pytest.approx(0.7) + glc_rxn = model.reactions.get_by_id("EX_glc") + assert glc_rxn.lower_bound == -8.0 + + +def test_condition_by_name(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1", "c2"], + p_tot=[0.5, 0.6], + gr_rate=[0.4, 0.7], + exch_fluxes=[[-5.0], [-8.0]], + exch_rxn_ids=["EX_glc"], + ) + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", condition="c2") + glc_rxn = model.reactions.get_by_id("EX_glc") + assert glc_rxn.lower_bound == -8.0 + + +# --------------------------------------------------------------------------- # +# Multi-rxn flux scenarios +# --------------------------------------------------------------------------- # + +def test_multiple_rxns_constrained_correctly(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-5.0, -2.0, 3.0]], + exch_rxn_ids=["EX_glc", "EX_O2", "EX_eth"], + ) + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", loose_strict_flux="loose") + glc = model.reactions.get_by_id("EX_glc") + o2 = model.reactions.get_by_id("EX_O2") + eth = model.reactions.get_by_id("EX_eth") + assert (glc.lower_bound, glc.upper_bound) == (-5.0, 0.0) + assert (o2.lower_bound, o2.upper_bound) == (-2.0, 0.0) + assert (eth.lower_bound, eth.upper_bound) == (0.0, 3.0) + + +def test_missing_c_source_is_ignored(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-5.0]], exch_rxn_ids=["EX_glc"], + ) + apply_flux_data_constraints( + model, fd, bio_rxn_id="biomass", c_source="EX_absent", + ) + assert model.reactions.get_by_id("EX_glc").lower_bound == -5.0 diff --git a/tests/test_experiments_data.py b/tests/test_experiments_data.py new file mode 100644 index 0000000..9a53c04 --- /dev/null +++ b/tests/test_experiments_data.py @@ -0,0 +1,66 @@ +"""Tests for experiments.data.""" +from raven_toolbox.experiments import ( + ExperimentData, + load_experiment_data, +) + + +_FLUX_DATA_TSV = ( + "Condition\tPtot\tgrRate\tglucose (r_1714)\tethanol (r_1761)\t" + "weight\tsource\n" + "glucose\tNaN\t0.36\t-13.33\t19.83\t1\tDLKcat\n" + "ethanol\tNaN\t0.12\tNaN\t-8.5\t1\tbrenda\n" +) + +_MAX_GROWTH_TSV = ( + "Condition\tPtot\tgrRate\tglucose (r_1714)\tfructose (r_1709)\t" + "weight\tsource\n" + "glucose\tNaN\t0.41\t-1000\tNaN\t1\tDLKcat\n" + "fructose\tNaN\t0.338\tNaN\t-1000\t1\tDLKcat\n" +) + +_ZERO_EXCH_TSV = "Rxns\nr_1542\nr_1545\nr_1546\n" + + +def test_load_experiment_data_all_files_present(tmp_path): + data_dir = tmp_path + (data_dir / "fluxData.tsv").write_text(_FLUX_DATA_TSV) + (data_dir / "maxGrowth.tsv").write_text(_MAX_GROWTH_TSV) + (data_dir / "zeroFlux.tsv").write_text(_ZERO_EXCH_TSV) + + experiment_data = load_experiment_data(tmp_path) + + assert isinstance(experiment_data, ExperimentData) + + assert experiment_data.flux_data is not None + assert experiment_data.flux_data.conds == ["glucose", "ethanol"] + assert experiment_data.flux_data.exch_rxn_ids == ["r_1714", "r_1761"] + assert experiment_data.flux_data.source == ["DLKcat", "brenda"] + assert list(experiment_data.flux_data.weight) == [1.0, 1.0] + + assert experiment_data.max_growth is not None + assert experiment_data.max_growth.conds == ["glucose", "fructose"] + assert experiment_data.max_growth.exch_rxn_ids == ["r_1714", "r_1709"] + + assert experiment_data.zero_flux == ["r_1542", "r_1545", "r_1546"] + + +def test_load_experiment_data_all_files_missing(tmp_path): + + experiment_data = load_experiment_data(tmp_path) + + assert experiment_data.flux_data is None + assert experiment_data.max_growth is None + assert experiment_data.zero_flux == [] + + +def test_load_experiment_data_partial(tmp_path): + """Only flux_data present: max_growth stays None.""" + data_dir = tmp_path + (data_dir / "fluxData.tsv").write_text(_FLUX_DATA_TSV) + + experiment_data = load_experiment_data(tmp_path) + + assert experiment_data.flux_data is not None + assert experiment_data.max_growth is None + assert experiment_data.zero_flux == [] diff --git a/tests/test_experiments_flux_data.py b/tests/test_experiments_flux_data.py new file mode 100644 index 0000000..d34590e --- /dev/null +++ b/tests/test_experiments_flux_data.py @@ -0,0 +1,283 @@ +"""Tests for load_flux_data.""" +from pathlib import Path + +import numpy as np +import pytest + +from raven_toolbox.experiments import FluxData, load_flux_data + + +# --------------------------------------------------------------------------- # +# Fixture builders +# --------------------------------------------------------------------------- # + +def _write_tsv(path: Path, rows: list[list[str]]) -> None: + path.write_text( + "\n".join("\t".join(r) for r in rows) + "\n", + encoding="utf-8", + ) + + +# --------------------------------------------------------------------------- # +# Pre-flight +# --------------------------------------------------------------------------- # + +def test_missing_file_raises(tmp_path): + with pytest.raises(FileNotFoundError, match="fluxData file"): + load_flux_data(tmp_path / "missing.tsv") + + +def test_empty_file_raises(tmp_path): + p = tmp_path / "empty.tsv" + p.write_text("", encoding="utf-8") + with pytest.raises(ValueError, match="empty"): + load_flux_data(p) + + +def test_short_header_raises(tmp_path): + """Header with fewer than 3 columns -> error.""" + p = tmp_path / "short.tsv" + _write_tsv(p, [["only", "two"]]) + with pytest.raises(ValueError, match="header has"): + load_flux_data(p) + + +def test_no_exchange_columns_raises(tmp_path): + """Header with exactly 3 columns (no exchanges) -> error.""" + p = tmp_path / "no_exch.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate"], + ["c1", "0.5", "0.4"], + ]) + with pytest.raises(ValueError, match="no exchange-flux columns"): + load_flux_data(p) + + +# --------------------------------------------------------------------------- # +# Basic happy path +# --------------------------------------------------------------------------- # + +def test_old_weight_column_name_is_rejected(tmp_path): + f = tmp_path / "fluxData.tsv" + _write_tsv(f, [ + ["Condition", "Ptot", "grRate", "glucose (r_1714)", "evotuneRMSEweight"], + ["c1", "0.5", "0.4", "-10", "1"], + ]) + with pytest.raises(ValueError, match="now called 'weight'"): + load_flux_data(f) + + +def test_weight_column_is_read_and_removed_from_the_fluxes(tmp_path): + f = tmp_path / "fluxData.tsv" + _write_tsv(f, [ + ["Condition", "Ptot", "grRate", "glucose (r_1714)", "weight", "source"], + ["c1", "0.5", "0.4", "-10", "0", "x"], + ["c2", "0.5", "0.3", "-8", "2", "y"], + ]) + data = load_flux_data(f) + assert data.weight.tolist() == [0.0, 2.0] + assert data.exch_rxn_ids == ["r_1714"] + + +def test_basic_single_condition(tmp_path): + p = tmp_path / "fd.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", "glucose (EX_glc)"], + ["c1", "0.5", "0.4", "-10.0"], + ]) + fd = load_flux_data(p) + assert isinstance(fd, FluxData) + assert fd.conds == ["c1"] + np.testing.assert_array_equal(fd.p_tot, [0.5]) + np.testing.assert_array_equal(fd.gr_rate, [0.4]) + np.testing.assert_array_equal(fd.exch_fluxes, [[-10.0]]) + assert fd.exch_mets == ["glucose"] + assert fd.exch_rxn_ids == ["EX_glc"] + + +def test_multi_condition_multi_exchange(tmp_path): + p = tmp_path / "fd.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", + "glucose (EX_glc)", "oxygen (EX_O2)", "ethanol (EX_eth)"], + ["c1", "0.5", "0.4", "-10.0", "-2.0", "5.0"], + ["c2", "0.6", "0.5", "-8.5", "-3.0", "7.0"], + ]) + fd = load_flux_data(p) + assert fd.conds == ["c1", "c2"] + np.testing.assert_array_equal(fd.p_tot, [0.5, 0.6]) + np.testing.assert_array_equal(fd.gr_rate, [0.4, 0.5]) + np.testing.assert_array_equal( + fd.exch_fluxes, + [[-10.0, -2.0, 5.0], [-8.5, -3.0, 7.0]], + ) + assert fd.exch_mets == ["glucose", "oxygen", "ethanol"] + assert fd.exch_rxn_ids == ["EX_glc", "EX_O2", "EX_eth"] + + +# --------------------------------------------------------------------------- # +# Exchange header parsing +# --------------------------------------------------------------------------- # + +def test_exchange_header_with_spaces_in_met_name(tmp_path): + p = tmp_path / "fd.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", "carbon dioxide (EX_co2)"], + ["c1", "0.5", "0.4", "5.0"], + ]) + fd = load_flux_data(p) + assert fd.exch_mets == ["carbon dioxide"] + assert fd.exch_rxn_ids == ["EX_co2"] + + +def test_exchange_header_without_parens_passes_through(tmp_path): + """Header without `(...)` falls back to met=rxn=full header.""" + p = tmp_path / "fd.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", "EX_glc"], + ["c1", "0.5", "0.4", "-10.0"], + ]) + fd = load_flux_data(p) + assert fd.exch_mets == ["EX_glc"] + assert fd.exch_rxn_ids == ["EX_glc"] + + +# --------------------------------------------------------------------------- # +# NaN handling +# --------------------------------------------------------------------------- # + +def test_empty_flux_cell_yields_nan(tmp_path): + p = tmp_path / "fd.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", "glc (EX_glc)", "o2 (EX_O2)"], + ["c1", "0.5", "0.4", "-10.0", ""], + ["c2", "0.6", "0.5", "", "-3.0"], + ]) + fd = load_flux_data(p) + assert np.isnan(fd.exch_fluxes[0, 1]) + assert np.isnan(fd.exch_fluxes[1, 0]) + assert fd.exch_fluxes[0, 0] == -10.0 + assert fd.exch_fluxes[1, 1] == -3.0 + + +def test_non_numeric_flux_yields_nan(tmp_path): + p = tmp_path / "fd.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", "glc (EX_glc)"], + ["c1", "0.5", "0.4", "NA"], + ]) + fd = load_flux_data(p) + assert np.isnan(fd.exch_fluxes[0, 0]) + + +# --------------------------------------------------------------------------- # +# Optional columns +# --------------------------------------------------------------------------- # + +def test_no_optional_columns_yields_none(tmp_path): + p = tmp_path / "fd.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", "glc (EX_glc)"], + ["c1", "0.5", "0.4", "-10.0"], + ]) + fd = load_flux_data(p) + assert fd.weight is None + assert fd.source is None + + +def test_weight_column_loaded(tmp_path): + p = tmp_path / "fd.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", + "glc (EX_glc)", "weight"], + ["c1", "0.5", "0.4", "-10.0", "1.5"], + ["c2", "0.6", "0.5", "-8.5", "0.7"], + ]) + fd = load_flux_data(p) + assert fd.weight is not None + np.testing.assert_array_equal(fd.weight, [1.5, 0.7]) + # Tuning-weight column dropped from exch_fluxes. + assert fd.exch_fluxes.shape == (2, 1) + assert fd.exch_rxn_ids == ["EX_glc"] + + +def test_source_column_loaded(tmp_path): + p = tmp_path / "fd.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", "glc (EX_glc)", "source"], + ["c1", "0.5", "0.4", "-10.0", "PMID:12345"], + ["c2", "0.6", "0.5", "-8.5", "in-house"], + ]) + fd = load_flux_data(p) + assert fd.source == ["PMID:12345", "in-house"] + assert fd.exch_fluxes.shape == (2, 1) + + +def test_both_optional_columns(tmp_path): + p = tmp_path / "fd.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", + "glc (EX_glc)", "weight", "source"], + ["c1", "0.5", "0.4", "-10.0", "1.5", "PMID:12345"], + ]) + fd = load_flux_data(p) + assert fd.weight is not None + np.testing.assert_array_equal(fd.weight, [1.5]) + assert fd.source == ["PMID:12345"] + assert fd.exch_fluxes.shape == (1, 1) + + +def test_optional_columns_in_arbitrary_position(tmp_path): + """Optional columns can appear before, between, or after the + exchange columns; they are pulled out independently of position.""" + p = tmp_path / "fd.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", + "source", "glc (EX_glc)", "weight"], + ["c1", "0.5", "0.4", "PMID:1", "-10.0", "1.5"], + ]) + fd = load_flux_data(p) + assert fd.source == ["PMID:1"] + assert fd.weight is not None + np.testing.assert_array_equal(fd.weight, [1.5]) + np.testing.assert_array_equal(fd.exch_fluxes, [[-10.0]]) + assert fd.exch_rxn_ids == ["EX_glc"] + + +# --------------------------------------------------------------------------- # +# Blank lines +# --------------------------------------------------------------------------- # + +def test_blank_lines_skipped(tmp_path): + p = tmp_path / "fd.tsv" + p.write_text( + "condition\tPtot\tgrRate\tglc (EX_glc)\n" + "\n" + "c1\t0.5\t0.4\t-10.0\n" + "\n", + encoding="utf-8", + ) + fd = load_flux_data(p) + assert fd.conds == ["c1"] + + +def test_ragged_row_raises(tmp_path): + p = tmp_path / "ragged.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", "Glc (EX_a)", "O2 (EX_b)"], + ["c1", "0.5", "0.4", "-1", "2"], + ["c2", "0.5", "0.4", "-1"], + ]) + with pytest.raises(ValueError, match="row 3 has 4 cell"): + load_flux_data(p) + + +def test_nested_parentheses_in_reaction_id(tmp_path): + p = tmp_path / "nested.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", "X (EX_a(e))", "O2 (EX_b)"], + ["c1", "0.5", "0.4", "-1", "2"], + ]) + fd = load_flux_data(p) + assert fd.exch_mets == ["X", "O2"] + assert fd.exch_rxn_ids == ["EX_a(e)", "EX_b"] From a63d279e4681ef38bbe553552a36d62a09938c2b Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Fri, 20 Nov 2026 21:53:56 +0100 Subject: [PATCH 2/6] feat: support optlang's scipy interface in utils.solver repair_scipy_problem, switch_to_scipy, edit_on_fast_solver, optimize_model and constraint_shadow_prices work around the dense coefficient matrix, the stale variable map and the missing solution status of the scipy interface, next to set_raven_tolerance. --- CHANGELOG.md | 4 + src/raven_toolbox/utils/solver.py | 678 ++++++++++++++++++++++++++++- tests/test_utils_optimize_model.py | 62 +++ tests/test_utils_solver_scipy.py | 256 +++++++++++ 4 files changed, 999 insertions(+), 1 deletion(-) create mode 100644 tests/test_utils_optimize_model.py create mode 100644 tests/test_utils_solver_scipy.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a249113..4779c3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ * New: the `experiments` package scores a model against measured growth rates and exchange fluxes. `load_experiment_data` reads `fluxData.tsv`, `maxGrowth.tsv` and `zeroFlux.tsv` from a folder, `load_flux_data` parses one flux file into a `FluxData` (with a per-condition `weight`), `simulate_conditions` fixes the uptake of each condition and solves the model, `compute_carbon_weights` gives each exchange the carbon count of its metabolite, `experiment_distance` combines the carbon-weighted RMSE of flux and max-growth conditions into one value, and `apply_flux_data_constraints` sets the exchange bounds of a model from one condition (loose, or a percentage band). The organism-specific steps, an anaerobic switch and a change of the protein content, and the solver call are passed as functions. +### Solver + +* New: `utils.solver` holds the support for optlang's scipy (HiGHS) interface, next to `set_raven_tolerance`. `repair_scipy_problem` renumbers the variable columns after a removal and rebuilds a stale coefficient matrix. `switch_to_scipy` and `edit_on_fast_solver` avoid the dense-matrix cost of building or editing a large model. `optimize_model` returns a solution with its status for an infeasible or unbounded LP instead of raising, and `constraint_shadow_prices` reads the row marginals on the scipy interface. + ### Data * The KEGG assets are the `kegg120` release of `toolbox-assets` (was: `kegg118`): the reference model and tables, the taxonomy and the profile HMM libraries of 4246 prokaryotic and 2724 eukaryotic KOs. The baked registry and `data/manifest.json` list them. diff --git a/src/raven_toolbox/utils/solver.py b/src/raven_toolbox/utils/solver.py index 245f1d9..b44618c 100644 --- a/src/raven_toolbox/utils/solver.py +++ b/src/raven_toolbox/utils/solver.py @@ -1,7 +1,86 @@ -"""Solver settings that RAVEN applies to every LP it solves.""" +"""Solver settings and support for optlang's scipy (HiGHS) solver interface. + +:data:`RAVEN_TOLERANCE` and :func:`set_raven_tolerance` give a model the tolerance that +RAVEN gives every LP it solves. + +The scipy interface of optlang differs from the other interfaces in ways that matter for a +large model, and the rest of this module works around them. + +Index map after a removal + ``scipy_interface.Problem`` keeps a map from variable name to column. After a variable is + removed it renumbers the columns with ``islice(variables, start, len(constraints))``, so + the last ``len(variables) - len(constraints)`` names keep their old column whenever the + problem has more variables than constraints. Primal values, and so every flux that cobra + reads, then come from the wrong columns or raise an ``IndexError``, and a later removal + of such a variable deletes the wrong column of the coefficient matrix. Variables are + removed lazily, by the next ``solver.update()``, which any ``add`` or ``optimize`` runs, so + the damage shows up in code that never removed anything itself. + + :func:`repair_scipy_problem` moves the problem to a subclass that renumbers correctly, so + every removal after it is right, and rebuilds the coefficient matrix from the constraint + expressions if the map was already stale. A model class that calls it whenever + ``model.solver`` is read is covered whatever removes the variables: the library, cobra, or + user code that calls ``model.remove_reactions``, ``reaction.delete`` or + ``model.remove_cons_vars``. A model that is built on a different interface and converted + afterwards with :func:`switch_to_scipy` has no removal on an unrepaired scipy problem. + For a plain ``cobra.Model`` on the scipy solver that you edit yourself, call ``repair_scipy_problem(model)`` before the first removal; a + call after the removals still restores the map and the matrix, but anything read in + between was wrong. + +Coefficient matrix + optlang stores the matrix densely and copies it on every added or removed variable or + constraint, so building or editing a genome-scale model (thousands of reactions) takes + hours. The subclass installed by :func:`repair_scipy_problem` keeps the coefficients in + dictionaries and hands HiGHS a sparse matrix, built once per structural change. + ``problem.A`` is then a ``scipy.sparse.csr_matrix`` (a dense array when there are no rows). + +Solution after a solve + ``cobra.Model.optimize`` sets the objective direction again after the solve, and the scipy + interface discards the solution on every direction assignment, also one that changes + nothing. ``Reaction.flux`` and ``Solver.primal_values`` then find no values after + ``model.optimize()``. The subclass discards the solution only when the direction changes. + + After a solve that finds no optimum (infeasible, unbounded) the scipy interface returns + ``None`` for every primal value and raises for the objective, so ``cobra.Model.optimize`` + fails with a ``TypeError`` instead of returning a solution with that status. The subclass + returns NaN for both, so the solution has the status and NaN values. + +Duals + The scipy interface raises for ``Constraint.dual`` and ``Variable.dual`` but keeps the + whole ``linprog`` result in ``problem._solution``. :func:`constraint_shadow_prices` reads + the row marginals from it, so the shadow prices of a scipy model equal those of the glpk + and gurobi interfaces. The stored result is used only after it is shown to be an optimal + primal and dual solution of the problem as it is now, so a result left over from before + a bound or an objective changed is never read. +""" from __future__ import annotations +import contextlib +from itertools import islice +from typing import Iterable + import cobra +import numpy as np +from scipy import sparse + +try: + from optlang import scipy_interface as _scipy_interface + from optlang.util import SolverTolerances + from scipy.optimize import linprog +except ImportError: # pragma: no cover - optlang always includes the interface + _scipy_interface = None + +__all__ = [ + "RAVEN_TOLERANCE", + "set_raven_tolerance", + "repair_scipy_problem", + "switch_to_scipy", + "fast_solver_while_building", + "edit_on_fast_solver", + "is_scipy_interface", + "constraint_shadow_prices", + "optimize_model", +] #: Feasibility and optimality tolerance that RAVEN's ``solveLP`` gives the #: solver (``optimizeProb``). cobrapy leaves the solver's own default, 1e-6 for @@ -18,3 +97,600 @@ def set_raven_tolerance(model: cobra.Model, tolerance: float = RAVEN_TOLERANCE) """ model.tolerance = tolerance return model + + +def optimize_model(model, **kwargs): + """``model.optimize()`` that returns the solution of an infeasible LP on every solver. + + Gurobi holds no primal values for an LP that it presolved as infeasible and optlang's + scipy interface returns ``None`` for them, and cobra reads them while it builds the + solution, so ``model.optimize()`` raises ``GurobiError`` or ``TypeError`` there. When + the solver status is not optimal this function returns a ``Solution`` with that status, + a NaN objective and NaN fluxes instead; with an optimal status every error is raised. + """ + try: + return model.optimize(**kwargs) + except Exception: + if model.solver.status == "optimal": + raise + import pandas as pd + from cobra.core import Solution + + return Solution( + objective_value=float("nan"), + status=model.solver.status, + fluxes=pd.Series(float("nan"), index=[r.id for r in model.reactions]), + ) + + +# Relative tolerance of the optimality certificate, scaled by the size of the quantity checked. +_CERTIFICATE_TOL = 1e-6 + + +if _scipy_interface is not None: + + class _SparseProblem(_scipy_interface.Problem): + """``scipy_interface.Problem`` with a sparse coefficient matrix. + + Renumbers every variable column after a removal and keeps its solution when the + direction is set to the value it has. + """ + + def __init__(self): + super().__init__() + self._init_sparse() + + # Feasibility and optimality tolerance of HiGHS; None keeps its default (1e-7). + _feasibility = None + _optimality = None + + def optimize(self, method="highs", verbosity=False, tolerance=1e-9, **kwargs): + """``linprog`` on the problem, with the tolerances that were set on the model.""" + c = np.array([self.objective.get(name, 0) for name in self._variables], dtype=float) + if self.direction == "max": + c *= -1 + options = {"maxiter": 10000, "disp": verbosity} + bounds = list(self.bounds.values()) + + def solve(options): + return linprog(c, self.A, self.upper_bounds, bounds=bounds, method=method, options=options, **kwargs) + + tight = dict(options) + if self._feasibility is not None: + tight["primal_feasibility_tolerance"] = self._feasibility + if self._optimality is not None: + tight["dual_feasibility_tolerance"] = self._optimality + solution = solve(tight) + # HiGHS reports numerical difficulties (status 4) on some LPs at a tolerance + # below its default; those are solved again at the default. + if solution.status == 4 and tight != options: + solution = solve(options) + self._solution = solution + self._status = solution.status + if self._status == 0: + self._var_primals = solution.x + self._slacks = solution.slack + else: + self._var_primals = None + self._slacks = None + self._f = solution.fun + + @property + def status(self): + """The solver status; a solve that ended with numerical difficulties is ``numeric``.""" + if self._status is None: + raise RuntimeError("Problem has not been optimized yet") + return _scipy_interface.SCIPY_STATUS.get(self._status, _scipy_interface.interface.NUMERIC) + + def _init_sparse(self): + # _rows[row name] = {variable name: coefficient}; _cols[variable name] = {row names} + self._rows = {} + self._cols = {name: set() for name in self._variables} + self._matrix = None + self._A = None + + @property + def direction(self): + return self._direction + + @direction.setter + def direction(self, value): + if value != self._direction: + self._direction = value + self._reset_solution() + + @property + def A(self): + """The linear coefficient matrix: sparse, or a dense array without rows.""" + n, m = len(self._variables), len(self._constraints) + if m == 0: + return np.zeros((0, n)) + if self._matrix is None: + data, rows, cols = [], [], [] + columns = self._variables + for row_name, row_index in self._constraints.items(): + for name, coefficient in self._rows[row_name].items(): + data.append(coefficient) + rows.append(row_index) + cols.append(columns[name]) + self._matrix = sparse.csr_matrix((data, (rows, cols)), shape=(m, n), dtype=float) + return self._matrix + + def get_var_primal(self, name): + """The primal value; NaN after a solve that found no optimum, None before any solve.""" + if self._var_primals is None: + return None if self._status is None else float("nan") + return self._var_primals[self._get_var_index(name)] + + @property + def objective_value(self): + """The optimum; NaN after a solve that found none.""" + if self._f is None and self._status is not None: + return float("nan") + return super().objective_value + + def add_variable(self, name): + if name in self._variables: + raise ValueError("A variable named " + name + " already exists.") + self._variables[name] = len(self._variables) + self.bounds[name] = (0, None) + self._cols[name] = set() + self._matrix = None + self._reset_solution() + + def add_constraint(self, name, coefficients={}, ub=0): + if name in self._constraints: + raise ValueError("A constraint named " + name + " already exists.") + self._constraints[name] = len(self._constraints) + self.upper_bounds = np.append(self.upper_bounds, ub) + row = {v: c for v, c in coefficients.items() if c != 0 and v in self._variables} + self._rows[name] = row + for v in row: + self._cols[v].add(name) + self._matrix = None + self._reset_solution() + + def remove_variable(self, name): + index = self._get_var_index(name) + for row_name in self._cols.pop(name): + del self._rows[row_name][name] + del self.bounds[name] + del self._variables[name] + self._update_variable_indices(index) + self._matrix = None + self._reset_solution() + + def remove_constraint(self, name): + index = self._get_constraint_index(name) + self.upper_bounds = np.delete(self.upper_bounds, index) + for v in self._rows.pop(name): + self._cols[v].discard(name) + del self._constraints[name] + self._update_constraint_indices(index) + self._matrix = None + self._reset_solution() + + def change_var_name(self, name, value): + super().change_var_name(name, value) + if name != value: + for row_name in self._cols[name]: + self._rows[row_name][value] = self._rows[row_name].pop(name) + self._cols[value] = self._cols.pop(name) + + def change_constraint_name(self, name, value): + super().change_constraint_name(name, value) + if name != value: + self._rows[value] = self._rows.pop(name) + for v in self._rows[value]: + self._cols[v].discard(name) + self._cols[v].add(value) + + def _update_variable_indices(self, start=0): + i = start + for name in islice(self._variables, start, None): + self._variables[name] = i + i += 1 + + +if _scipy_interface is not None: + + class _SparseModel(_scipy_interface.Model): + """``scipy_interface.Model`` on a `_SparseProblem` that copies and pickles sparsely. + + optlang pickles a model as JSON and restores it by adding the variables and + constraints to a fresh dense problem, which takes tens of minutes for a genome-scale + model. This class restores into a `_SparseProblem`, so ``copy.deepcopy``, + ``cobra.Model.copy`` and the worker processes of a pool get their copy in seconds. + """ + + # optlang finds its interface module (the scipy one) through this attribute. + __module__ = _scipy_interface.__name__ + + def _initialize_problem(self): + self.problem = _SparseProblem() + + def _remove_constraints(self, constraints): + # optlang's scipy interface calls the generic removal inside its loop, once per + # constraint with the whole list, so removing two or more constraints at once + # (up to 350) raises ``LookupError: Constraint ... not in solver``. + for constraint in constraints: + if constraint.lb is not None: + self.problem.remove_constraint(constraint.lower_constraint_name) + if constraint.ub is not None: + self.problem.remove_constraint(constraint.upper_constraint_name) + _scipy_interface.interface.Model._remove_constraints(self, constraints) + + def _initialize_configuration(self): + super()._initialize_configuration() + _install_tolerances(self) + + def __reduce__(self): + problem = self.problem + return (_restore_sparse_model, (self.to_json(), problem._feasibility, problem._optimality)) + + def _restore_sparse_model(state, feasibility, optimality): + model = _SparseModel() + model._init_from_json(state) + model.problem._feasibility = feasibility + model.problem._optimality = optimality + return model + + def _install_tolerances(solver): + """Let ``solver.configuration.tolerances`` (and so ``cobra.Model.tolerance``) set the + feasibility and optimality tolerance of HiGHS, which optlang's scipy interface ignores.""" + + def setter(attribute): + return lambda value: setattr(solver.problem, attribute, float(value)) + + solver.configuration.tolerances = SolverTolerances( + { + "feasibility": (lambda: solver.problem._feasibility or 1e-7, setter("_feasibility")), + "optimality": (lambda: solver.problem._optimality or 1e-7, setter("_optimality")), + } + ) + + +def is_scipy_interface(interface) -> bool: + """True if ``interface`` (a solver name, an optlang interface module or a model's + ``problem`` attribute) is optlang's scipy interface.""" + if _scipy_interface is None: + return False + if isinstance(interface, str): + return interface.lower() == "scipy" + return interface is _scipy_interface + + +def _is_scipy_problem(problem) -> bool: + return _scipy_interface is not None and isinstance(problem, _scipy_interface.Problem) + + +def _map_is_stale(problem) -> bool: + return list(problem._variables.values()) != list(range(len(problem._variables))) + + +def _adopt_dense_matrix(problem) -> None: + """Fill the dictionaries of a `_SparseProblem` from the dense matrix it was swapped from.""" + dense = problem._A + names = list(problem._variables) + row_names = list(problem._constraints) + problem._init_sparse() + problem._rows = {name: {} for name in row_names} + for r, c in zip(*np.nonzero(dense)): + problem._rows[row_names[r]][names[c]] = float(dense[r, c]) + problem._cols[names[c]].add(row_names[r]) + + +def _rebuild_from_expressions(problem, solver) -> None: + """Fill the dictionaries of a `_SparseProblem` from the constraint expressions of ``solver``. + + The expressions are the source the matrix is derived from, so this is correct whatever + happened to the matrix itself. + """ + problem._init_sparse() + problem._rows = {name: {} for name in problem._constraints} + for i, name in enumerate(problem._variables): + problem._variables[name] = i + for constraint in solver.constraints: + coefficients = None + for row_name, sign in ( + (constraint.upper_constraint_name, 1.0), + (constraint.lower_constraint_name, -1.0), + ): + if row_name not in problem._rows: + continue + if coefficients is None: + coefficients = constraint.coefficient_dict() + row = {v: sign * c for v, c in coefficients.items() if c != 0 and v in problem._variables} + problem._rows[row_name] = row + for v in row: + problem._cols[v].add(row_name) + + +def repair_scipy_problem(model) -> bool: + """Make the scipy problem of ``model`` correct and fast to edit. + + Does nothing and returns False when the solver of ``model`` is not optlang's scipy + interface. Otherwise it moves the problem to a class that renumbers the variable columns + correctly after a removal and stores the coefficients sparsely (see the module docstring), + applies the pending changes, restores the column map and the coefficient matrix if a + removal had already left the map stale, and returns True. Once the problem is switched, a + call is a type check. + + Parameters + ---------- + model + A ``cobra.Model``. A model class can call this + function already; call it yourself for a plain ``cobra.Model`` before you remove + variables (``remove_reactions``, ``Reaction.delete``, ``remove_cons_vars``). + """ + solver = getattr(model, "_solver", None) + problem = getattr(solver, "problem", None) + if not _is_scipy_problem(problem): + return False + if isinstance(problem, _SparseProblem): + return True + stale = _map_is_stale(problem) + problem.__class__ = _SparseProblem + solver.__class__ = _SparseModel + _install_tolerances(solver) + if stale: + problem._init_sparse() + problem._rows = {name: {} for name in problem._constraints} + else: + _adopt_dense_matrix(problem) + solver.update() + if stale: + _rebuild_from_expressions(problem, solver) + return True + + +def switch_to_scipy(model) -> None: + """Move ``model`` to the scipy solver, fast, with the sparse problem of this module. + + ``model.solver = "scipy"`` clones the problem through optlang, which grows a dense matrix + row by row. This builds the problem the way ``cobra.Model`` does when it is created, on + the sparse class, so a genome-scale model switches in seconds. The objective and its + direction are kept. A model whose solver holds anything besides one forward and one + reverse variable per reaction and one constraint per metabolite (an extra constraint or + variable of your own) is switched with optlang's clone instead and then repaired. + """ + if repair_scipy_problem(model): + return + solver = model.solver + plain = len(solver.constraints) == len(model.metabolites) and len(solver.variables) == 2 * len( + model.reactions + ) + if _scipy_interface is None or not plain: + from cobra import Model + + Model.solver.fset(model, "scipy") + repair_scipy_problem(model) + return + from cobra.util.solver import linear_reaction_coefficients + from optlang.symbolics import Zero + + coefficients = linear_reaction_coefficients(model) + direction = model.objective.direction + new = _SparseModel() + new.objective = _scipy_interface.Objective(Zero) + model._solver = new + _populate_sparse(model, new) + model.objective = coefficients + model.objective.direction = direction + if getattr(model, "_tolerance", None) is not None: + try: + model.tolerance = model._tolerance + except Exception: # the scipy interface has no tolerance settings to receive it + pass + + +def _populate_sparse(model, solver) -> None: + """Give ``solver`` one forward and one reverse variable per reaction of ``model`` and one + constraint per metabolite, as ``cobra.Model._populate_solver`` does. + + ``_populate_solver`` sets the coefficients of a constraint one at a time, and the scipy + interface removes and adds the rows of a constraint for each call, which takes hours for a + genome-scale model. Here every constraint is created with its whole expression and added + once. + """ + from optlang.symbolics import Zero, add + + problem = solver.interface + variables = {} + for reaction in model.reactions: + variables[reaction.id] = (problem.Variable(reaction.id), problem.Variable(reaction.reverse_id)) + solver.add([v for pair in variables.values() for v in pair]) + terms = {metabolite.id: [] for metabolite in model.metabolites} + for reaction in model.reactions: + forward, reverse = variables[reaction.id] + for metabolite, coefficient in reaction.metabolites.items(): + terms[metabolite.id].append(coefficient * forward) + terms[metabolite.id].append(-coefficient * reverse) + solver.add( + [ + problem.Constraint(add(*expressions) if expressions else Zero, name=name, lb=0, ub=0) + for name, expressions in terms.items() + ] + ) + for reaction in model.reactions: + reaction.update_variable_bounds() + + +@contextlib.contextmanager +def edit_on_fast_solver(model): + """Run structural edits of ``model`` on glpk when it is on the scipy solver. + + Removing reactions and metabolites, or rewriting the coefficients of thousands of + reactions, is quick on glpk and slow on the scipy interface, where optlang rewrites the + constraint expressions (and the rows) for each edit. Inside the block a scipy model is on + glpk; on exit it is converted back with :func:`switch_to_scipy`. The block changes nothing + for a model on another solver, for a model inside a ``with model:`` context (its + rollbacks hold the solver objects) and when glpk is unavailable. The optlang solver of + the model is a different object inside the block and after it. + """ + from cobra import Model + + solver = getattr(model, "_solver", None) + if not _is_scipy_problem(getattr(solver, "problem", None)) or getattr(model, "_contexts", None): + yield False + return + try: + Model.solver.fset(model, "glpk") + except Exception: + yield False + return + try: + yield True + finally: + switch_to_scipy(model) + + +@contextlib.contextmanager +def fast_solver_while_building(): + """Make models created inside the block on a solver that builds large models quickly. + + When cobra's default solver is scipy the block runs with glpk as the default, which + builds a genome-scale model in seconds; the default is restored on exit. Models created + inside are on glpk; convert them with :func:`switch_to_scipy`. With any other default + solver, or when glpk is unavailable, the block changes nothing. + """ + import cobra + + config = cobra.Configuration() + previous = config.solver + switched = False + if getattr(previous, "__name__", str(previous)).endswith("scipy_interface"): + try: + config.solver = "glpk" + switched = True + except Exception: + switched = False + try: + yield switched + finally: + if switched: + config.solver = previous + + +def _objective_vector(problem) -> np.ndarray: + """Cost vector ``linprog`` minimises: the objective, negated for a maximisation.""" + cost = np.array([problem.objective.get(name, 0.0) for name in problem._variables], dtype=float) + if problem.direction == "max": + cost = -cost + return cost + + +def _certified_result(problem): + """The stored ``linprog`` result if it is an optimum of ``problem`` as it is now, else None. + + Checks primal feasibility against the current rows and bounds, stationarity of the cost + vector against the stored marginals, and equality of the primal and dual objective. A + result that passes is an optimal primal-dual pair of the current problem, whatever solve + produced it. + """ + result = getattr(problem, "_solution", None) + try: + x = np.asarray(result.x, dtype=float) + y = np.asarray(result.ineqlin.marginals, dtype=float) + z_low = np.asarray(result.lower.marginals, dtype=float) + z_up = np.asarray(result.upper.marginals, dtype=float) + fun = float(result.fun) + matrix = sparse.csr_matrix(problem.A, dtype=float) + rhs = np.asarray(problem.upper_bounds, dtype=float) + bounds = list(problem.bounds.values()) + except (AttributeError, TypeError, ValueError): + return None + n, m = len(problem._variables), len(problem._constraints) + if matrix.shape != (m, n) or not (x.shape == z_low.shape == z_up.shape == (n,)) or y.shape != (m,): + return None + if len(bounds) != n or not np.all(np.isfinite(x)): + return None + low = np.array([-np.inf if b[0] is None else b[0] for b in bounds], dtype=float) + up = np.array([np.inf if b[1] is None else b[1] for b in bounds], dtype=float) + cost = _objective_vector(problem) + tol = _CERTIFICATE_TOL + + def close(a, b, scale): + return abs(a - b) <= tol * (1.0 + scale) + + if m and np.any(matrix @ x - rhs > tol * (1.0 + np.abs(rhs))): + return None + if np.any(x < low - tol * (1.0 + np.abs(low))) or np.any(x > up + tol * (1.0 + np.abs(up))): + return None + reduced = cost - (matrix.T @ y if m else 0.0) + if np.max(np.abs(reduced - (z_low + z_up)), initial=0.0) > tol * (1.0 + np.max(np.abs(cost), initial=0.0)): + return None + primal = float(cost @ x) + dual = float(rhs @ y) if m else 0.0 + dual += float(np.sum(low[np.isfinite(low)] * z_low[np.isfinite(low)])) + dual += float(np.sum(up[np.isfinite(up)] * z_up[np.isfinite(up)])) + if not (close(primal, fun, abs(fun)) and close(primal, dual, abs(primal))): + return None + return result + + +def constraint_shadow_prices(model, constraint_ids: Iterable[str]) -> np.ndarray: + """Shadow prices of the named constraints, on every solver, as one array. + + The shadow price of a constraint is the change of the optimum, in the direction of the + model objective, per unit rise of its right-hand side: for a metabolite balance + ``S v = 0`` it is the optimum gained per unit of the metabolite supplied. It is the value + of ``cobra.Metabolite.shadow_price`` for the glpk, gurobi and cplex interfaces, and for the + scipy interface (which raises) it is computed from the HiGHS row marginals of the last + solve. + + The model must have been solved. On the scipy interface the stored result is used only if + it is an optimum of the problem as it is now; otherwise the model is solved once more. If + that solve is not optimal, the function raises. + + Parameters + ---------- + model + A ``cobra.Model`` that has been optimized. + constraint_ids + IDs of constraints of the model, for example the metabolite ids of a pool constraint. + + Returns + ------- + numpy.ndarray + One shadow price per ID, in the order given. + + Raises + ------ + RuntimeError + If the scipy problem has no optimal solution to read the marginals from. + KeyError + If an ID is not a constraint of the model. + """ + ids = list(constraint_ids) + constraints = model.constraints + repair_scipy_problem(model) + problem = getattr(model.solver, "problem", None) + if _scipy_interface is None or not isinstance(problem, _scipy_interface.Problem): + return np.array([float(constraints[cid].dual) for cid in ids], dtype=float) + + result = _certified_result(problem) + if result is None: + model.slim_optimize() + result = _certified_result(problem) + if result is None: + raise RuntimeError( + "shadow prices need an optimal solution of the scipy problem; the model has none " + f"(solver status {model.solver.status!r})" + ) + marginals = np.asarray(result.ineqlin.marginals, dtype=float) + rows = problem._constraints + # The rows hold `A x <= ub` and `-A x <= -lb`. The marginal of a row is d(minimised value)/d(rhs + # of that row), so d/d(ub) is the upper row's marginal and d/d(lb) is minus the lower row's. + # A maximisation minimises the negated objective. + sign = -1.0 if problem.direction == "max" else 1.0 + prices = np.zeros(len(ids), dtype=float) + for k, cid in enumerate(ids): + constraint = constraints[cid] + value = 0.0 + if constraint.ub is not None: + value += marginals[rows[constraint.upper_constraint_name]] + if constraint.lb is not None: + value -= marginals[rows[constraint.lower_constraint_name]] + prices[k] = sign * value + return prices diff --git a/tests/test_utils_optimize_model.py b/tests/test_utils_optimize_model.py new file mode 100644 index 0000000..06804da --- /dev/null +++ b/tests/test_utils_optimize_model.py @@ -0,0 +1,62 @@ +"""Tests for optimize_model.""" +import math + +import cobra +import pytest + +from raven_toolbox.utils.solver import optimize_model + + +def _infeasible_model(solver: str) -> cobra.Model: + model = cobra.Model("m") + a = cobra.Metabolite("A", compartment="c") + rxn = cobra.Reaction("R1") + rxn.add_metabolites({a: 1.0}) + rxn.bounds = (0.0, 10.0) + sink = cobra.Reaction("S1") + sink.add_metabolites({a: -1.0}) + sink.bounds = (20.0, 30.0) + model.add_reactions([rxn, sink]) + model.objective = "R1" + model.solver = solver + return model + + +@pytest.mark.parametrize("solver", ["glpk", "scipy"]) +def test_infeasible_lp_gives_an_infeasible_solution(solver): + solution = optimize_model(_infeasible_model(solver)) + assert solution.status == "infeasible" + + +def test_feasible_lp_matches_model_optimize(): + model = _infeasible_model("glpk") + model.reactions.get_by_id("S1").bounds = (0.0, 5.0) + solution = optimize_model(model) + assert solution.status == "optimal" + assert solution.objective_value == pytest.approx(5.0) + + +def test_gurobi_presolved_infeasible_lp_gives_an_infeasible_solution(): + """Gurobi holds no primal values for an infeasible LP that it presolved, which makes + ``model.optimize()`` raise ``GurobiError``.""" + try: + model = _infeasible_model("gurobi") + except Exception as exc: + pytest.skip(f"gurobi unusable: {exc}") + model.solver.problem.Params.Presolve = 2 + solution = optimize_model(model) + assert solution.status == "infeasible" + assert math.isnan(solution.objective_value) + assert set(solution.fluxes.index) == {"R1", "S1"} + + +def test_other_errors_are_raised(): + class Broken: + class solver: + status = "optimal" + + def optimize(self, **kwargs): + raise RuntimeError("boom") + + with pytest.raises(RuntimeError, match="boom"): + optimize_model(Broken()) diff --git a/tests/test_utils_solver_scipy.py b/tests/test_utils_solver_scipy.py new file mode 100644 index 0000000..23d2ae5 --- /dev/null +++ b/tests/test_utils_solver_scipy.py @@ -0,0 +1,256 @@ +"""Tests for the scipy (HiGHS) support in raven_toolbox.utils.solver. + +optlang's scipy interface keeps a variable index map that goes wrong after a removal, +which makes cobra read fluxes from the wrong columns, and it raises for duals. +raven_toolbox.utils.solver repairs the first and reads the HiGHS marginals for the second. +""" +import cobra +import numpy as np +import pytest + +from raven_toolbox.utils.solver import ( + constraint_shadow_prices, + edit_on_fast_solver, + repair_scipy_problem, + switch_to_scipy, +) + +@pytest.fixture +def scipy_default(): + """cobra builds every new model on scipy while the test runs.""" + config = cobra.Configuration() + previous = config.solver + config.solver = "scipy" + yield + config.solver = previous + + +def _columns_are_consistent(model): + problem = model.solver.problem + return list(problem._variables.values()) == list(range(len(problem._variables))) + + +def _ring(solver): + """A plain cobra model on ``solver`` with more variables than constraints.""" + config = cobra.Configuration() + previous = config.solver + config.solver = solver + try: + model = cobra.Model("plain") + mets = [cobra.Metabolite(f"m{i}") for i in range(3)] + rxns = [] + for i in range(6): + rxn = cobra.Reaction(f"r{i}") + rxn.bounds = (0, 10) + rxn.add_metabolites({mets[i % 3]: 1.0, mets[(i + 1) % 3]: -1.0} if i else {mets[0]: 1.0}) + rxns.append(rxn) + model.add_reactions(rxns) + finally: + config.solver = previous + model.objective = "r5" + return model + + +def test_a_plain_cobra_model_is_repaired_on_request(): + model, reference = _ring("scipy"), _ring("glpk") + for m in (model, reference): + m.remove_reactions(["r1", "r2"]) + m.reactions.get_by_id("r4").bounds = (0, 3) + assert repair_scipy_problem(model) + assert _columns_are_consistent(model) + assert model.slim_optimize() == pytest.approx(reference.slim_optimize()) + assert model.reactions.get_by_id("r5").flux == pytest.approx(reference.reactions.get_by_id("r5").flux) + + +def test_repair_does_nothing_on_other_solvers(): + model = cobra.Model("m") + model.solver = "glpk" + assert repair_scipy_problem(model) is False + + +def _columns_ring(solver): + """Three metabolites, twelve reactions: removals leave late variables with stale columns.""" + config = cobra.Configuration() + previous = config.solver + config.solver = solver + try: + model = cobra.Model("ring") + mets = [cobra.Metabolite(f"m{i}") for i in range(3)] + rxns = [] + for i in range(12): + rxn = cobra.Reaction(f"r{i}") + rxn.bounds = (0, 10 + i) + if i == 0: + rxn.add_metabolites({mets[0]: 1.0}) + else: + rxn.add_metabolites({mets[i % 3]: -1.0, mets[(i + 1) % 3]: 0.5 + 0.1 * i}) + rxns.append(rxn) + model.add_reactions(rxns) + finally: + config.solver = previous + model.objective = "r11" + return model + + +def test_removals_after_the_install_keep_every_column_right(): + model, reference = _columns_ring("scipy"), _columns_ring("glpk") + assert repair_scipy_problem(model) + for rid in ("r3", "r7", "r9", "r10"): + model.reactions.get_by_id(rid).delete() + reference.reactions.get_by_id(rid).delete() + assert _columns_are_consistent(model) + assert model.slim_optimize() == pytest.approx(reference.slim_optimize()) + solution = model.optimize() + expected = reference.optimize() + assert solution.objective_value == pytest.approx(expected.objective_value) + for rid in solution.fluxes.index: + assert model.reactions.get_by_id(rid).flux == pytest.approx(solution.fluxes[rid]) + + +def test_a_map_that_is_already_stale_is_rebuilt_from_the_expressions(): + model, reference = _columns_ring("scipy"), _columns_ring("glpk") + model.reactions.get_by_id("r5").delete() + reference.reactions.get_by_id("r5").delete() + model.solver.update() + assert not _columns_are_consistent(model) + assert repair_scipy_problem(model) + assert _columns_are_consistent(model) + assert model.slim_optimize() == pytest.approx(reference.slim_optimize()) + + +def test_the_sparse_problem_hands_highs_the_matrix_of_the_dense_problem(): + model = _columns_ring("scipy") + dense = model.solver.problem.A.copy() + assert repair_scipy_problem(model) + assert (model.solver.problem.A.toarray() == dense).all() + + +def test_switch_to_scipy_keeps_the_objective_and_its_direction(): + model, reference = _columns_ring("glpk"), _columns_ring("glpk") + for m in (model, reference): + m.objective = "r8" + m.objective_direction = "min" + m.reactions.get_by_id("r1").lower_bound = 2.0 + switch_to_scipy(model) + assert model.solver.interface.__name__.endswith("scipy_interface") + assert model.objective_direction == "min" + assert model.slim_optimize() == pytest.approx(reference.slim_optimize()) + assert model.reactions.get_by_id("r8").flux == pytest.approx(model.slim_optimize()) + + +@pytest.mark.parametrize("status_case", ["infeasible", "unbounded"]) +def test_a_solve_without_optimum_returns_a_solution_with_its_status(status_case): + config = cobra.Configuration() + previous = config.solver + config.solver = "scipy" + try: + model = cobra.Model("chain") + met = cobra.Metabolite("m") + source, sink = cobra.Reaction("source"), cobra.Reaction("sink") + source.add_metabolites({met: 1.0}) + sink.add_metabolites({met: -1.0}) + source.bounds = (0, 10) if status_case == "infeasible" else (0, 1000.0) + sink.bounds = (20, 30) if status_case == "infeasible" else (0, 1000.0) + model.add_reactions([source, sink]) + finally: + config.solver = previous + model.objective = "sink" + if status_case == "unbounded": + source.upper_bound = float("inf") + sink.upper_bound = float("inf") + assert repair_scipy_problem(model) + if status_case == "unbounded": + # cobra raises for this status on every solver + with pytest.raises(cobra.exceptions.OptimizationError, match="unbounded"): + model.optimize(raise_error=False) + return + solution = model.optimize(raise_error=False) + assert solution.status == status_case + assert np.isnan(solution.objective_value) + assert solution.fluxes.isna().all() + + +def test_removing_several_constraints_at_once_on_the_sparse_problem(): + model = _columns_ring("scipy") + repair_scipy_problem(model) + ids = [m.id for m in list(model.metabolites)[:3]] + model.remove_cons_vars([model.constraints[i] for i in ids]) + model.solver.update() + assert len(model.solver.problem._constraints) == 2 * len(model.constraints) + assert not set(ids) & set(model.constraints) + + +def test_edit_on_fast_solver_returns_to_scipy_with_the_same_problem(): + model = _columns_ring("scipy") + repair_scipy_problem(model) + before = model.slim_optimize() + with edit_on_fast_solver(model) as fast: + assert fast + assert model.solver.interface.__name__.endswith("glpk_interface") + model.reactions.get_by_id("r1").bounds = (0.0, 5.0) + assert model.solver.interface.__name__.endswith("scipy_interface") + assert model.reactions.get_by_id("r1").upper_bound == 5.0 + assert model.slim_optimize() <= before + reference = _columns_ring("glpk") + reference.reactions.get_by_id("r1").bounds = (0.0, 5.0) + assert model.slim_optimize() == pytest.approx(reference.slim_optimize()) + + +def test_edit_on_fast_solver_changes_nothing_on_glpk(): + model = _columns_ring("glpk") + with edit_on_fast_solver(model) as fast: + assert not fast + assert model.solver.interface.__name__.endswith("glpk_interface") + + +def test_model_tolerance_reaches_highs_and_survives_a_copy(monkeypatch): + import pickle + + from scipy import optimize + + model = _ring("scipy") + repair_scipy_problem(model) + model.tolerance = 1e-9 + assert model.solver.configuration.tolerances.feasibility == 1e-9 + assert model.solver.configuration.tolerances.optimality == 1e-9 + + seen = [] + real = optimize.linprog + + def spy(*args, **kwargs): + seen.append(kwargs["options"]) + return real(*args, **kwargs) + + for m in (model, pickle.loads(pickle.dumps(model)), model.copy()): + monkeypatch.setattr("raven_toolbox.utils.solver.linprog", spy) + m.slim_optimize() + assert len(seen) == 3 + assert all(o["primal_feasibility_tolerance"] == 1e-9 for o in seen) + assert all(o["dual_feasibility_tolerance"] == 1e-9 for o in seen) + + +def test_a_solve_with_numerical_difficulties_is_repeated_at_the_default_tolerance(monkeypatch): + from scipy import optimize + + model = _ring("scipy") + repair_scipy_problem(model) + expected = model.slim_optimize() + model.tolerance = 1e-9 + real = optimize.linprog + calls = [] + + def numerical_trouble_when_tight(*args, **kwargs): + calls.append(kwargs["options"]) + if "primal_feasibility_tolerance" in kwargs["options"]: + return optimize.OptimizeResult(status=4, fun=None, x=None, slack=None) + return real(*args, **kwargs) + + monkeypatch.setattr("raven_toolbox.utils.solver.linprog", numerical_trouble_when_tight) + assert model.slim_optimize() == pytest.approx(expected) + assert len(calls) == 2 + + def always_numerical_trouble(*args, **kwargs): + return optimize.OptimizeResult(status=4, fun=None, x=None, slack=None) + + monkeypatch.setattr("raven_toolbox.utils.solver.linprog", always_numerical_trouble) + assert model.optimize(raise_error=False).status == "numeric" From ebb592001ad8d1255ae3eacc8d893ca289347e53 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Fri, 20 Nov 2026 21:53:56 +0100 Subject: [PATCH 3/6] feat: add calculate_mw and find_met_smiles calculate_mw gives the molecular weight of a protein from its sequence. find_met_smiles fills the SMILES annotation of metabolites from PubChem, with a local cache file. --- CHANGELOG.md | 5 + src/raven_toolbox/annotation/__init__.py | 5 +- src/raven_toolbox/annotation/smiles.py | 233 ++++++++++++++ src/raven_toolbox/utils/protein.py | 113 +++++++ tests/test_annotation_smiles.py | 389 +++++++++++++++++++++++ tests/test_utils_protein.py | 168 ++++++++++ 6 files changed, 912 insertions(+), 1 deletion(-) create mode 100644 src/raven_toolbox/annotation/smiles.py create mode 100644 src/raven_toolbox/utils/protein.py create mode 100644 tests/test_annotation_smiles.py create mode 100644 tests/test_utils_protein.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4779c3c..1c6b2be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,11 @@ * New: `utils.solver` holds the support for optlang's scipy (HiGHS) interface, next to `set_raven_tolerance`. `repair_scipy_problem` renumbers the variable columns after a removal and rebuilds a stale coefficient matrix. `switch_to_scipy` and `edit_on_fast_solver` avoid the dense-matrix cost of building or editing a large model. `optimize_model` returns a solution with its status for an infeasible or unbounded LP instead of raising, and `constraint_shadow_prices` reads the row marginals on the scipy interface. +### Proteins and annotation + +* New: `utils.protein.calculate_mw` computes the molecular weight of a protein from its amino acid sequence (average residue masses plus one water; the codes B, J, O, U, X and Z are recognised; an empty sequence gives NaN). +* New: `annotation.find_met_smiles` fills `annotation["smiles"]` of the metabolites from PubChem, with a local TSV cache (`cache_path`) that is appended to after each lookup, and `annotation.met_smiles` reads the SMILES of a metabolite whether the annotation is a string or a list. + ### Data * The KEGG assets are the `kegg120` release of `toolbox-assets` (was: `kegg118`): the reference model and tables, the taxonomy and the profile HMM libraries of 4246 prokaryotic and 2724 eukaryotic KOs. The baked registry and `data/manifest.json` list them. diff --git a/src/raven_toolbox/annotation/__init__.py b/src/raven_toolbox/annotation/__init__.py index 7b0cc5b..b0a742a 100644 --- a/src/raven_toolbox/annotation/__init__.py +++ b/src/raven_toolbox/annotation/__init__.py @@ -1,4 +1,4 @@ -"""Annotation helpers: SBO term assignment and ΔG CSV persistence. +"""Annotation helpers: SBO term assignment, ΔG CSV persistence and SMILES lookup. These are the parts of yeast-GEM's ``missingFields`` module that are organism-agnostic. Default parameter values @@ -7,6 +7,7 @@ overrides. """ from raven_toolbox.annotation.delta_g import load_delta_g_csv, save_delta_g_csv +from raven_toolbox.annotation.smiles import find_met_smiles, met_smiles from raven_toolbox.annotation.sbo import ( DEFAULT_BIOMASS_MET_NAMES, DEFAULT_BIOMASS_RXN_NAME, @@ -19,6 +20,8 @@ "DEFAULT_BIOMASS_RXN_NAME", "DEFAULT_NGAM_RXN_NAME", "add_sbo_terms", + "find_met_smiles", "load_delta_g_csv", + "met_smiles", "save_delta_g_csv", ] diff --git a/src/raven_toolbox/annotation/smiles.py b/src/raven_toolbox/annotation/smiles.py new file mode 100644 index 0000000..7aa5bb5 --- /dev/null +++ b/src/raven_toolbox/annotation/smiles.py @@ -0,0 +1,233 @@ +"""SMILES lookup via PubChem with a local TSV cache. + +The cache file is a two-column TSV with no header: ``namesmiles``. +Each successful lookup is appended immediately so interrupted runs +resume gracefully. +""" +from __future__ import annotations + +import logging +import re +import time +from pathlib import Path +from typing import TYPE_CHECKING +from urllib.parse import quote + +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +if TYPE_CHECKING: + import cobra + + +__all__ = ["find_met_smiles", "met_smiles"] + +logger = logging.getLogger(__name__) + +# PubChem deprecated the `CanonicalSMILES` property in 2025 in favour of +# `SMILES` (its canonical/absolute SMILES); `ConnectivitySMILES` replaces +# the old `IsomericSMILES`. +_PUBCHEM_URL = ( + "https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/name/" + "{name}/property/SMILES/TXT" +) +_TIMEOUT_SECONDS = 30 +_NUM_RETRIES = 10 +_INTER_REQUEST_SLEEP = 0.05 # seconds; PubChem allows ~5 req/s +_PROT_PATTERN = re.compile(r"^prot_") +_FIRST_LINE_PATTERN = re.compile(r"^(\S*)\n") + + +def met_smiles(metabolite) -> str: + """The metabolite's SMILES, whatever shape its annotation is in. + + cobra stores an annotation value as the object it was assigned, and + returns a *list* for a model read back from a file: a metabolite + annotated in memory by :func:`find_met_smiles` has ``"CCO"``, + and the same metabolite after it is saved and loaded has + ``["CCO"]``. Readers that assume a string get a list that compares + equal to nothing and is not hashable. + + Returns the first non-empty entry, or ``""`` when there is none. + """ + value = metabolite.annotation.get("smiles", "") + if isinstance(value, (list, tuple, set)): + return next((str(v) for v in value if v), "") + return str(value) if value else "" + + +def find_met_smiles( + model: "cobra.Model", + *, + cache_path: str | Path, +) -> None: + """Populate ``metabolite.annotation['smiles']`` from PubChem. + + For every metabolite in the model: + + 1. Skip if the name starts with ``prot_`` (enzyme pseudometabolite). + 2. Skip, with a warning, if the metabolite has no name. + 3. Skip if ``annotation['smiles']`` is already set. + 4. Look up the metabolite name in the local TSV cache. When the cache + holds a name more than once, the last row is used. + 5. If still not found, query PubChem; cache the result. + + The same metabolite name may appear on many ``cobra.Metabolite`` + instances (different compartments). Lookup is done once per unique + name and the SMILES is applied to every metabolite sharing that name. + + Parameters + ---------- + model + A cobra.Model. Mutated in place: each metabolite gains + ``annotation['smiles']`` when a SMILES is found. + cache_path + Path to the SMILES cache TSV. The file is created if missing and + appended to as new SMILES are downloaded. + """ + cache_path = Path(cache_path) + cache_path.parent.mkdir(parents=True, exist_ok=True) + + # Build map: metabolite name -> list of metabolite objects sharing it. + name_to_mets: dict[str, list] = {} + unnamed = [] + for met in model.metabolites: + name = met.name + if not name: + unnamed.append(met.id) + continue + if _PROT_PATTERN.match(name): + continue + if met.annotation.get("smiles"): + continue + name_to_mets.setdefault(name, []).append(met) + + if unnamed: + logger.warning( + "find_met_smiles: %d metabolite(s) have no name and get no SMILES: %s", + len(unnamed), ", ".join(unnamed), + ) + + if not name_to_mets: + logger.info("find_met_smiles: nothing to look up.") + return + + # Load existing cache. + cache = _load_cache(cache_path) + logger.info( + "find_met_smiles: %d unique metabolite name(s) need SMILES; " + "%d already in cache.", + len(name_to_mets), + sum(1 for n in name_to_mets if n in cache), + ) + + # Apply cached SMILES first (no network). + for name in list(name_to_mets): + if name in cache: + smiles = cache[name] + for met in name_to_mets[name]: + if smiles: + met.annotation["smiles"] = smiles + del name_to_mets[name] + + if not name_to_mets: + logger.info("find_met_smiles: all matches resolved from cache.") + return + + # Hit PubChem for the rest. + session = _make_session() + n_found = 0 + n_missing = 0 + + for name in sorted(name_to_mets): + smiles = _fetch_one_smiles(session, name) + # Append to cache immediately, including empty results so we + # do not retry next time. + _append_cache(cache_path, name, smiles) + + if smiles: + for met in name_to_mets[name]: + met.annotation["smiles"] = smiles + n_found += 1 + else: + n_missing += 1 + logger.debug("No SMILES for: %s", name) + + time.sleep(_INTER_REQUEST_SLEEP) + + total = n_found + n_missing + if total > 0: + pct = 100.0 * n_found / total + logger.info( + "find_met_smiles: SMILES found for %d/%d unique names (%.0f%%); " + "%d missing.", + n_found, total, pct, n_missing, + ) + + +def _load_cache(path: Path) -> dict[str, str]: + """Load the two-column TSV cache. Returns an empty dict if missing.""" + cache: dict[str, str] = {} + if not path.is_file(): + return cache + with open(path, "r", encoding="utf-8") as f: + for line in f: + parts = line.rstrip("\n").split("\t") + if len(parts) >= 2: + cache[parts[0]] = parts[1] + elif len(parts) == 1 and parts[0]: + cache[parts[0]] = "" + return cache + + +def _append_cache(path: Path, name: str, smiles: str) -> None: + """Append one row to the TSV cache. Open and close per call so + interrupted runs leave a consistent file.""" + with open(path, "a", encoding="utf-8") as f: + f.write(f"{name}\t{smiles}\n") + + +def _make_session() -> requests.Session: + """A requests.Session with retry/backoff for transient errors.""" + session = requests.Session() + retry = Retry( + total=_NUM_RETRIES, + backoff_factor=0.5, + status_forcelist=(429, 500, 502, 503, 504), + allowed_methods=("GET",), + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount("https://", adapter) + session.mount("http://", adapter) + return session + + +def _fetch_one_smiles(session: requests.Session, name: str) -> str: + """Fetch SMILES from PubChem for one metabolite name. Returns + empty string if unfound or unrecoverable error.""" + url = _PUBCHEM_URL.format(name=quote(name, safe="")) + try: + response = session.get(url, timeout=_TIMEOUT_SECONDS) + except requests.RequestException as e: + logger.warning("PubChem request failed for %r: %s", name, e) + return "" + + if response.status_code in (400, 404, 500): + return "" + if response.status_code != 200: + logger.warning( + "PubChem returned HTTP %d for %r; treating as no match.", + response.status_code, name, + ) + return "" + + text = response.text + if not text: + return "" + + # Multiple SMILES may come on separate lines; take only the first. + match = _FIRST_LINE_PATTERN.match(text) + if match: + return match.group(1).strip() + return text.split("\n", 1)[0].strip() diff --git a/src/raven_toolbox/utils/protein.py b/src/raven_toolbox/utils/protein.py new file mode 100644 index 0000000..a87ea97 --- /dev/null +++ b/src/raven_toolbox/utils/protein.py @@ -0,0 +1,113 @@ +"""Compute the molecular weight of a protein from its amino acid sequence.""" +from __future__ import annotations + +import logging + +__all__ = ["calculate_mw"] + +logger = logging.getLogger(__name__) + + +# Average mass of water (g/mol). Added once to account for the two +# H atoms and one OH that remain after peptide bond condensation. +_WATER_MASS = 18.01528 + + +# Average residue masses (g/mol). Standard amino acids agree with +# widely-cited references (e.g., ExPASy ProtParam). Non-standard codes: +# - B = average(D, N). +# - J = same as L and I, which both happen to equal 113.16. +# - O (pyrrolysine) = 255.31. +# - U (selenocysteine) = 150.04. +# - Z = average(E, Q). +# +# X (any amino acid): the unweighted mean of the 20 standard residues +# above, 118.885. +_STANDARD_RESIDUE_MASSES: dict[str, float] = { + "A": 71.08, + "C": 103.14, + "D": 115.09, + "E": 129.11, + "F": 147.17, + "G": 57.05, + "H": 137.14, + "I": 113.16, + "K": 128.17, + "L": 113.16, + "M": 131.20, + "N": 114.10, + "P": 97.12, + "Q": 128.13, + "R": 156.19, + "S": 87.08, + "T": 101.10, + "V": 99.13, + "W": 186.21, + "Y": 163.17, +} + +_X_MASS = sum(_STANDARD_RESIDUE_MASSES.values()) / len(_STANDARD_RESIDUE_MASSES) + +_RESIDUE_MASSES: dict[str, float] = { + **_STANDARD_RESIDUE_MASSES, + "B": (115.09 + 114.10) / 2, # D-or-N + "J": 113.16, # L-or-I (both 113.16) + "O": 255.31, # pyrrolysine + "U": 150.04, # selenocysteine + "X": _X_MASS, # any standard residue + "Z": (129.11 + 128.13) / 2, # E-or-Q +} + + +def calculate_mw(sequence: str) -> float: + """Compute the molecular weight of a protein from its sequence. + + Each residue contributes its average mass; one water mass is added + once (representing the H + OH at the peptide termini, equivalent to + one un-eliminated water relative to N residues + N water masses). + + Recognized amino acid codes (the sequence is uppercased first, so + matching is case-insensitive): + + ACDEFGHIKLMNPQRSTVWY - the 20 standard residues + BJOZUX - non-standard / ambiguous codes + + Whitespace and digits are skipped silently. Any other unrecognized + character is also skipped, but triggers one warning per call + listing every distinct offending character. + + Parameters + ---------- + sequence + Protein sequence as a string. May contain whitespace, digits, + and other non-letter characters; those are ignored. + + Returns + ------- + float + Molecular weight in g/mol (Da). Returns ``nan`` for an + empty/all-skipped sequence (no residues), so a missing sequence + is not mistaken for an 18 Da "protein". + """ + upper = sequence.upper() if sequence else "" + mw = _WATER_MASS + n_residues = 0 + + unknown_chars: set[str] = set() + for ch in upper: + if ch in _RESIDUE_MASSES: + mw += _RESIDUE_MASSES[ch] + n_residues += 1 + elif not ch.isspace() and not ch.isdigit(): + unknown_chars.add(ch) + + if unknown_chars: + logger.warning( + "calculate_mw: skipped %d unknown character(s): %s", + len(unknown_chars), + ", ".join(repr(c) for c in sorted(unknown_chars)), + ) + + if n_residues == 0: + return float("nan") + return mw diff --git a/tests/test_annotation_smiles.py b/tests/test_annotation_smiles.py new file mode 100644 index 0000000..c9f780c --- /dev/null +++ b/tests/test_annotation_smiles.py @@ -0,0 +1,389 @@ +"""Tests for find_met_smiles (network mocked).""" +import logging + +import cobra + +from raven_toolbox.annotation import find_met_smiles + + +# --------------------------------------------------------------------------- # +# Fake HTTP layer +# --------------------------------------------------------------------------- # + +class _FakeResponse: + def __init__(self, status_code: int, text: str = ""): + self.status_code = status_code + self.text = text + + +def _install_fake_session(monkeypatch, name_to_response: dict): + """Replace _make_session so URL fetches return canned responses. + + `name_to_response` maps URL-escaped metabolite names to + (status_code, text). Names not in the map return 404. + """ + from raven_toolbox.annotation import smiles as mod + + class _FakeSession: + def get(self, url, timeout=None): + for fragment, (status, text) in name_to_response.items(): + if fragment in url: + return _FakeResponse(status, text) + return _FakeResponse(404, "") + + monkeypatch.setattr(mod, "_make_session", lambda: _FakeSession()) + # Skip the inter-request sleep in tests. + monkeypatch.setattr(mod, "_INTER_REQUEST_SLEEP", 0.0) + + +def _build_model(metabolites: list[tuple[str, str]]) -> cobra.Model: + """Build a cobra.Model from (id, name) tuples.""" + model = cobra.Model("test") + for met_id, met_name in metabolites: + m = cobra.Metabolite(met_id, name=met_name, compartment="c") + model.add_metabolites([m]) + return model + + +def _smiles(model: cobra.Model, met_id: str) -> str: + return model.metabolites.get_by_id(met_id).annotation.get("smiles", "") + + +# --------------------------------------------------------------------------- # +# Cache hits +# --------------------------------------------------------------------------- # + +def test_uses_cache_without_network(monkeypatch, tmp_path): + """If a SMILES is in the cache, no network call happens.""" + cache_path = tmp_path / "smilesDB.tsv" + cache_path.write_text("glucose\tOC1OC(CO)C(O)C(O)C1O\n") + + model = _build_model([("glc_c", "glucose")]) + + def _fail(): + raise AssertionError("network should not be called") + + from raven_toolbox.annotation import smiles as mod + monkeypatch.setattr(mod, "_make_session", _fail) + + find_met_smiles(model, cache_path=cache_path) + assert _smiles(model, "glc_c") == "OC1OC(CO)C(O)C(O)C1O" + + +def test_cache_with_empty_smiles_skips_network(monkeypatch, tmp_path): + """A cached empty result means 'we already know there is no SMILES'. + Should not retry the network for it.""" + cache_path = tmp_path / "smilesDB.tsv" + cache_path.write_text("unknown_compound\t\n") + + model = _build_model([("u_c", "unknown_compound")]) + + def _fail(): + raise AssertionError("network should not be called") + + from raven_toolbox.annotation import smiles as mod + monkeypatch.setattr(mod, "_make_session", _fail) + + find_met_smiles(model, cache_path=cache_path) + assert _smiles(model, "u_c") == "" + + +# --------------------------------------------------------------------------- # +# Network lookups +# --------------------------------------------------------------------------- # + +def test_fetches_from_network_when_cache_missing(monkeypatch, tmp_path): + cache_path = tmp_path / "smilesDB.tsv" + _install_fake_session(monkeypatch, { + "glucose": (200, "OC1OC(CO)C(O)C(O)C1O\n"), + }) + + model = _build_model([("glc_c", "glucose")]) + find_met_smiles(model, cache_path=cache_path) + + assert _smiles(model, "glc_c") == "OC1OC(CO)C(O)C(O)C1O" + + +def test_takes_first_line_when_multiple_returned(monkeypatch, tmp_path): + """PubChem may return multiple SMILES separated by newlines; the + function should keep only the first.""" + cache_path = tmp_path / "smilesDB.tsv" + _install_fake_session(monkeypatch, { + "ambiguous": (200, "C1CCCCC1\nCCCCCCC\nCCCC\n"), + }) + + model = _build_model([("a_c", "ambiguous")]) + find_met_smiles(model, cache_path=cache_path) + + assert _smiles(model, "a_c") == "C1CCCCC1" + + +def test_404_means_no_smiles(monkeypatch, tmp_path): + cache_path = tmp_path / "smilesDB.tsv" + _install_fake_session(monkeypatch, {}) # everything 404s + + model = _build_model([("x_c", "unknown_compound")]) + find_met_smiles(model, cache_path=cache_path) + + assert _smiles(model, "x_c") == "" + + +def test_500_treated_as_no_smiles(monkeypatch, tmp_path): + """A 500 (often returned for slashes etc. in names) is treated + as 'no match' rather than a crash.""" + cache_path = tmp_path / "smilesDB.tsv" + _install_fake_session(monkeypatch, { + "weird/name": (500, ""), + }) + model = _build_model([("w_c", "weird/name")]) + find_met_smiles(model, cache_path=cache_path) + assert _smiles(model, "w_c") == "" + + +# --------------------------------------------------------------------------- # +# Cache write-back +# --------------------------------------------------------------------------- # + +def test_appends_to_cache_after_fetch(monkeypatch, tmp_path): + cache_path = tmp_path / "smilesDB.tsv" + _install_fake_session(monkeypatch, { + "glucose": (200, "OC1OC(CO)C(O)C(O)C1O\n"), + }) + model = _build_model([("glc_c", "glucose")]) + find_met_smiles(model, cache_path=cache_path) + + text = cache_path.read_text() + assert "glucose\tOC1OC(CO)C(O)C(O)C1O\n" in text + + +def test_caches_empty_result_too(monkeypatch, tmp_path): + """Even when nothing is found, the empty result is cached so we + do not retry next time.""" + cache_path = tmp_path / "smilesDB.tsv" + _install_fake_session(monkeypatch, {}) + + model = _build_model([("x_c", "unknown")]) + find_met_smiles(model, cache_path=cache_path) + + text = cache_path.read_text() + assert "unknown\t\n" in text + + +def test_existing_cache_preserved_when_appending(monkeypatch, tmp_path): + """Existing cache entries should not be lost.""" + cache_path = tmp_path / "smilesDB.tsv" + cache_path.write_text("water\tO\n") + + _install_fake_session(monkeypatch, { + "ethanol": (200, "CCO\n"), + }) + model = _build_model([ + ("h2o_c", "water"), + ("etoh_c", "ethanol"), + ]) + find_met_smiles(model, cache_path=cache_path) + + text = cache_path.read_text() + assert "water\tO\n" in text + assert "ethanol\tCCO\n" in text + + +# --------------------------------------------------------------------------- # +# Filters and skip conditions +# --------------------------------------------------------------------------- # + +def test_skips_prot_metabolites(monkeypatch, tmp_path): + """Metabolite names starting with 'prot_' must not be looked up.""" + cache_path = tmp_path / "smilesDB.tsv" + + requested_urls: list[str] = [] + + class _CaptureSession: + def get(self, url, timeout=None): + requested_urls.append(url) + return _FakeResponse(404, "") + + from raven_toolbox.annotation import smiles as mod + monkeypatch.setattr(mod, "_make_session", lambda: _CaptureSession()) + monkeypatch.setattr(mod, "_INTER_REQUEST_SLEEP", 0.0) + + model = _build_model([ + ("prot_P1_c", "prot_P1"), + ("prot_pool_c", "prot_pool"), + ]) + find_met_smiles(model, cache_path=cache_path) + + assert requested_urls == [] + + +def test_skips_metabolites_with_existing_smiles(monkeypatch, tmp_path): + """If a metabolite already has annotation['smiles'], do not overwrite.""" + cache_path = tmp_path / "smilesDB.tsv" + _install_fake_session(monkeypatch, { + "glucose": (200, "should_not_be_used\n"), + }) + + model = _build_model([("glc_c", "glucose")]) + model.metabolites.get_by_id("glc_c").annotation["smiles"] = "PRESERVE_ME" + + find_met_smiles(model, cache_path=cache_path) + + assert _smiles(model, "glc_c") == "PRESERVE_ME" + + +def test_same_name_in_multiple_compartments_gets_same_smiles( + monkeypatch, tmp_path, +): + """Glucose in cytoplasm and extracellular compartments share a + name and therefore one PubChem lookup; both metabolites should + receive the SMILES.""" + cache_path = tmp_path / "smilesDB.tsv" + _install_fake_session(monkeypatch, { + "glucose": (200, "OC1OC(CO)C(O)C(O)C1O\n"), + }) + + model = cobra.Model("test") + m_c = cobra.Metabolite("glc_c", name="glucose", compartment="c") + m_e = cobra.Metabolite("glc_e", name="glucose", compartment="e") + model.add_metabolites([m_c, m_e]) + + find_met_smiles(model, cache_path=cache_path) + + assert _smiles(model, "glc_c") == "OC1OC(CO)C(O)C(O)C1O" + assert _smiles(model, "glc_e") == "OC1OC(CO)C(O)C(O)C1O" + + +def test_empty_model_does_not_crash(monkeypatch, tmp_path): + cache_path = tmp_path / "smilesDB.tsv" + model = cobra.Model("empty") + find_met_smiles(model, cache_path=cache_path) + + +# --------------------------------------------------------------------------- # +# Cache file creation +# --------------------------------------------------------------------------- # + +def test_creates_cache_file_if_missing(monkeypatch, tmp_path): + cache_path = tmp_path / "subdir" / "smilesDB.tsv" + _install_fake_session(monkeypatch, { + "glucose": (200, "C\n"), + }) + model = _build_model([("glc_c", "glucose")]) + find_met_smiles(model, cache_path=cache_path) + assert cache_path.is_file() + + +# --------------------------------------------------------------------------- # +# URL escaping +# --------------------------------------------------------------------------- # + +def test_url_escapes_special_characters(monkeypatch, tmp_path): + """A name with spaces or slashes should be URL-encoded before being + inserted into the URL.""" + cache_path = tmp_path / "smilesDB.tsv" + captured: list[str] = [] + + class _CaptureSession: + def get(self, url, timeout=None): + captured.append(url) + return _FakeResponse(404, "") + + from raven_toolbox.annotation import smiles as mod + monkeypatch.setattr(mod, "_make_session", lambda: _CaptureSession()) + monkeypatch.setattr(mod, "_INTER_REQUEST_SLEEP", 0.0) + + model = _build_model([("x", "L-alanyl-L-glycine")]) + find_met_smiles(model, cache_path=cache_path) + + assert len(captured) == 1 + # Hyphens are URL-safe; we just check the name is in the URL exactly + # (no spaces, no broken escaping). + assert "L-alanyl-L-glycine" in captured[0] + + +def test_url_escapes_spaces_and_slashes(monkeypatch, tmp_path): + cache_path = tmp_path / "smilesDB.tsv" + captured: list[str] = [] + + class _CaptureSession: + def get(self, url, timeout=None): + captured.append(url) + return _FakeResponse(404, "") + + from raven_toolbox.annotation import smiles as mod + monkeypatch.setattr(mod, "_make_session", lambda: _CaptureSession()) + monkeypatch.setattr(mod, "_INTER_REQUEST_SLEEP", 0.0) + + model = _build_model([("x", "weird name/here")]) + find_met_smiles(model, cache_path=cache_path) + + assert len(captured) == 1 + assert " " not in captured[0] + # Unescaped slash would still be valid in a URL but PubChem would + # interpret it as a path separator. Our quote(safe='') should + # escape it. + assert "weird%20name%2Fhere" in captured[0] + + +# --------------------------------------------------------------------------- # +# Logging +# --------------------------------------------------------------------------- # + +def test_logs_summary_after_run(monkeypatch, tmp_path, caplog): + cache_path = tmp_path / "smilesDB.tsv" + _install_fake_session(monkeypatch, { + "glucose": (200, "C\n"), + }) + model = _build_model([ + ("glc_c", "glucose"), + ("u_c", "unknown_compound"), + ]) + + with caplog.at_level(logging.INFO): + find_met_smiles(model, cache_path=cache_path) + + assert "SMILES found for" in caplog.text + + +# --------------------------------------------------------------------------- # +# Reading the annotation back +# --------------------------------------------------------------------------- # + +def test_met_smiles_accepts_the_shape_a_loaded_model_carries(): + """cobra returns a list for a model read from a file, a string for one + annotated in memory. Both mean the same molecule.""" + import cobra + + from raven_toolbox.annotation import met_smiles + + met = cobra.Metabolite("etoh") + met.annotation["smiles"] = "CCO" + assert met_smiles(met) == "CCO" + + met.annotation["smiles"] = ["CCO"] + assert met_smiles(met) == "CCO" + + met.annotation["smiles"] = [] + assert met_smiles(met) == "" + + del met.annotation["smiles"] + assert met_smiles(met) == "" + + +def test_duplicate_cache_name_uses_the_last_row(monkeypatch, tmp_path): + cache_path = tmp_path / "smilesDB.tsv" + cache_path.write_text("glucose\tFIRST\nglucose\tLAST\n") + model = _build_model([("glc_c", "glucose")]) + find_met_smiles(model, cache_path=cache_path) + assert _smiles(model, "glc_c") == "LAST" + + +def test_metabolite_without_name_is_skipped_with_a_warning(monkeypatch, tmp_path, caplog): + cache_path = tmp_path / "smilesDB.tsv" + cache_path.write_text("m1_c\tFROM_ID\nglucose\tCCO\n") + model = _build_model([("m1_c", ""), ("glc_c", "glucose")]) + with caplog.at_level(logging.WARNING): + find_met_smiles(model, cache_path=cache_path) + assert _smiles(model, "m1_c") == "" + assert _smiles(model, "glc_c") == "CCO" + assert any("m1_c" in r.getMessage() and "no name" in r.getMessage() for r in caplog.records) diff --git a/tests/test_utils_protein.py b/tests/test_utils_protein.py new file mode 100644 index 0000000..0364543 --- /dev/null +++ b/tests/test_utils_protein.py @@ -0,0 +1,168 @@ +"""Tests for calculate_mw.""" +import logging +import math + +import pytest + +from raven_toolbox.utils.protein import ( + calculate_mw, + _RESIDUE_MASSES, + _STANDARD_RESIDUE_MASSES, + _WATER_MASS, + _X_MASS, +) + + +# --------------------------------------------------------------------------- # +# Trivial cases +# --------------------------------------------------------------------------- # + +def test_empty_sequence_returns_nan(): + # An empty / all-skipped sequence has no residues; NaN avoids being + # mistaken for an 18 Da "protein". + assert math.isnan(calculate_mw("")) + assert math.isnan(calculate_mw(" ")) + assert math.isnan(calculate_mw("123")) + + +def test_single_residue(): + assert calculate_mw("A") == pytest.approx(_WATER_MASS + 71.08) + + +def test_dipeptide(): + expected = _WATER_MASS + 71.08 + 137.14 + assert calculate_mw("AH") == pytest.approx(expected) + + +# --------------------------------------------------------------------------- # +# Known reference cases +# --------------------------------------------------------------------------- # + +def test_methionine_only(): + """Met has mass 131.20.""" + assert calculate_mw("M") == pytest.approx(_WATER_MASS + 131.20) + + +def test_repeated_residues_count_correctly(): + expected = _WATER_MASS + 5 * 71.08 + assert calculate_mw("AAAAA") == pytest.approx(expected) + + +def test_all_20_standard_residues(): + """A sequence of one of each of the 20 standard residues should + give water + sum of all 20 masses.""" + seq = "".join(sorted(_STANDARD_RESIDUE_MASSES.keys())) + expected = _WATER_MASS + sum(_STANDARD_RESIDUE_MASSES.values()) + assert calculate_mw(seq) == pytest.approx(expected) + + +# --------------------------------------------------------------------------- # +# Non-standard codes +# --------------------------------------------------------------------------- # + +def test_b_is_average_of_d_and_n(): + expected = (_RESIDUE_MASSES["D"] + _RESIDUE_MASSES["N"]) / 2 + assert _RESIDUE_MASSES["B"] == pytest.approx(expected) + + +def test_z_is_average_of_e_and_q(): + expected = (_RESIDUE_MASSES["E"] + _RESIDUE_MASSES["Q"]) / 2 + assert _RESIDUE_MASSES["Z"] == pytest.approx(expected) + + +def test_j_equals_l_and_i(): + assert _RESIDUE_MASSES["J"] == _RESIDUE_MASSES["L"] == _RESIDUE_MASSES["I"] + + +def test_o_pyrrolysine(): + assert _RESIDUE_MASSES["O"] == pytest.approx(255.31) + + +def test_u_selenocysteine(): + assert _RESIDUE_MASSES["U"] == pytest.approx(150.04) + + +def test_x_is_mean_of_20_standard_residues(): + expected = sum(_STANDARD_RESIDUE_MASSES.values()) / 20 + assert _X_MASS == pytest.approx(expected) + assert _X_MASS == pytest.approx(118.885, abs=1e-3) + + +# --------------------------------------------------------------------------- # +# Case insensitivity +# --------------------------------------------------------------------------- # + +def test_lowercase_is_treated_same_as_uppercase(): + assert calculate_mw("ahkc") == calculate_mw("AHKC") + + +def test_mixed_case_works(): + assert calculate_mw("AhKc") == calculate_mw("AHKC") + + +# --------------------------------------------------------------------------- # +# Whitespace and digits ignored silently +# --------------------------------------------------------------------------- # + +def test_whitespace_ignored(): + assert calculate_mw("A H") == calculate_mw("AH") + assert calculate_mw("\tA\nH\n") == calculate_mw("AH") + + +def test_digits_ignored(): + assert calculate_mw("A1H2") == calculate_mw("AH") + + +def test_whitespace_and_digits_do_not_warn(caplog): + with caplog.at_level(logging.WARNING): + calculate_mw("A H 1 2 K") + assert "skipped" not in caplog.text + + +# --------------------------------------------------------------------------- # +# Unknown characters: warn but do not raise +# --------------------------------------------------------------------------- # + +def test_unknown_character_warns(caplog): + with caplog.at_level(logging.WARNING): + calculate_mw("A@H") + assert "skipped" in caplog.text + assert "@" in caplog.text + + +def test_unknown_character_does_not_contribute_to_mass(caplog): + with caplog.at_level(logging.WARNING): + result = calculate_mw("A@H") + assert result == calculate_mw("AH") + + +def test_multiple_unknown_chars_warned_together(caplog): + with caplog.at_level(logging.WARNING): + calculate_mw("A@H#K?L") + assert "skipped" in caplog.text + assert "@" in caplog.text + assert "#" in caplog.text + assert "?" in caplog.text + + +# --------------------------------------------------------------------------- # +# Realistic sequence +# --------------------------------------------------------------------------- # + +def test_short_real_protein_mass(): + """A snippet of a real protein sequence should yield a plausible + mass in the low-kDa range, not just any positive number.""" + seq = "MENTGTLAGCDVRFGRSGGSGSGSGSAATAACK" + mw = calculate_mw(seq) + assert mw > 0 + # Approximately 33 residues at average ~110 g/mol = 3630 + water. + assert 3000 < mw < 4500 + + +# --------------------------------------------------------------------------- # +# Water mass precision +# --------------------------------------------------------------------------- # + +def test_water_mass_is_precise(): + """The water mass used is the precise average, 18.01528 Da.""" + assert _WATER_MASS == pytest.approx(18.01528) From 17528ce7dff8a714f9e167b5d7e1e7060d51ec6a Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Fri, 20 Nov 2026 21:53:56 +0100 Subject: [PATCH 4/6] feat: fit GAM and NGAM from chemostat and retentostat data fit_maintenance fits NGAM from a maintenance coefficient and GAM from the substrate uptake of chemostats, with a study-weighted Huber loss and a bootstrap over studies. set_gam takes keep_offsets_of and get_gam reads the coefficient. --- CHANGELOG.md | 5 + src/raven_toolbox/biomass/__init__.py | 30 +- src/raven_toolbox/biomass/gam.py | 56 +++- src/raven_toolbox/biomass/maintenance.py | 357 +++++++++++++++++++++++ tests/test_biomass_maintenance.py | 292 ++++++++++++++++++ 5 files changed, 734 insertions(+), 6 deletions(-) create mode 100644 src/raven_toolbox/biomass/maintenance.py create mode 100644 tests/test_biomass_maintenance.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c6b2be..84ef756 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,11 @@ * New: `utils.solver` holds the support for optlang's scipy (HiGHS) interface, next to `set_raven_tolerance`. `repair_scipy_problem` renumbers the variable columns after a removal and rebuilds a stale coefficient matrix. `switch_to_scipy` and `edit_on_fast_solver` avoid the dense-matrix cost of building or editing a large model. `optimize_model` returns a solution with its status for an infeasible or unbounded LP instead of raising, and `constraint_shadow_prices` reads the row marginals on the scipy interface. +### Biomass + +* New: `biomass.fit_maintenance` fits NGAM from a maintenance coefficient and GAM from chemostat substrate uptake, with a study-weighted Huber loss and a bootstrap over studies. The steps are also available separately (`fit_gam`, `bootstrap_gam`, `uptake_grid`, `min_uptake`, `ngam_from_maintenance_coefficient`, `study_weights`, `huber_loss`, `log_ratio`), and `MaintenanceConfig` holds the reaction ids and metabolite names. +* New: `get_gam`, and `set_gam` takes `keep_offsets_of`: the coefficient of that metabolite becomes the value and the other GAM terms shift by the same amount, so a charge-balancing proton offset stays. + ### Proteins and annotation * New: `utils.protein.calculate_mw` computes the molecular weight of a protein from its amino acid sequence (average residue masses plus one water; the codes B, J, O, U, X and Z are recognised; an empty sequence gives NaN). diff --git a/src/raven_toolbox/biomass/__init__.py b/src/raven_toolbox/biomass/__init__.py index c0fc62a..12f12c3 100644 --- a/src/raven_toolbox/biomass/__init__.py +++ b/src/raven_toolbox/biomass/__init__.py @@ -26,6 +26,9 @@ scale_biomass(model, cfg, "protein", 0.50, balance_out="carbohydrate") set_gam(model, 80, biomass_rxn=cfg.biomass_rxn, cofactor_met_names=("ATP", "ADP", "H2O", "H+", "phosphate")) + +GAM and NGAM are fitted to chemostat uptakes and a maintenance coefficient with +:func:`fit_maintenance` (see :mod:`raven_toolbox.biomass.maintenance`). """ from raven_toolbox.biomass.config import BiomassComponent, BiomassConfig from raven_toolbox.biomass.fit_parameters import ( @@ -33,7 +36,20 @@ ParameterPosition, fit_parameters, ) -from raven_toolbox.biomass.gam import set_gam +from raven_toolbox.biomass.gam import get_gam, set_gam +from raven_toolbox.biomass.maintenance import ( + MaintenanceConfig, + MaintenanceFit, + bootstrap_gam, + fit_gam, + fit_maintenance, + huber_loss, + log_ratio, + min_uptake, + ngam_from_maintenance_coefficient, + study_weights, + uptake_grid, +) from raven_toolbox.biomass.scale import ( rescale_pseudoreaction, scale_biomass, @@ -44,10 +60,22 @@ "BiomassComponent", "BiomassConfig", "FitParametersResult", + "MaintenanceConfig", + "MaintenanceFit", "ParameterPosition", + "bootstrap_gam", + "fit_gam", + "fit_maintenance", "fit_parameters", + "get_gam", + "huber_loss", + "log_ratio", + "min_uptake", + "ngam_from_maintenance_coefficient", "rescale_pseudoreaction", "scale_biomass", "set_gam", + "study_weights", "sum_biomass", + "uptake_grid", ] diff --git a/src/raven_toolbox/biomass/gam.py b/src/raven_toolbox/biomass/gam.py index d054576..9f5d8cf 100644 --- a/src/raven_toolbox/biomass/gam.py +++ b/src/raven_toolbox/biomass/gam.py @@ -1,6 +1,9 @@ """Growth-associated maintenance (GAM) and non-growth maintenance (NGAM). -Ports the generic part of yeast-GEM's ``changeGAM.m``. +Ports the generic part of yeast-GEM's ``changeGAM.m``. GAM is the ATP hydrolysed per gram +of biomass formed (mmol/gDW), held in the coefficients of the ATP hydrolysis terms of the +biomass pseudoreaction. NGAM is the ATP hydrolysed per hour and gram of biomass +(mmol/gDW/h), held as the fixed flux of the maintenance reaction. """ from __future__ import annotations @@ -17,6 +20,7 @@ def set_gam( cofactor_met_names: Iterable[str], ngam_rxn: str | None = None, ngam_value: float | None = None, + keep_offsets_of: str | None = None, ) -> cobra.Model: """Set GAM (and optionally NGAM) on a model in place. @@ -25,6 +29,12 @@ def set_gam( H+, phosphate) to ``±value``, keeping the sign of its current coefficient. + With ``keep_offsets_of`` the coefficients keep their differences instead: + the coefficient of the metabolite of that name becomes ``±value`` and every + other cofactor changes by the same amount in the direction of its sign. A + proton coefficient that balances the charge of the other biomass terms then + keeps its offset from the ATP coefficient. + If both ``ngam_rxn`` and ``ngam_value`` are given, the NGAM reaction's bounds are fixed at ``(ngam_value, ngam_value)``. An empty ``ngam_rxn`` counts as not given, and ``ngam_value`` without an ``ngam_rxn`` is an @@ -43,6 +53,11 @@ def set_gam( Iterable of metabolite names (not ids). Every metabolite in the biomass pseudoreaction whose name is in this set is rescaled. yeast-GEM uses ``{"ATP", "ADP", "H2O", "H+", "phosphate"}``. + keep_offsets_of + Name of the metabolite whose coefficient becomes ``±value`` while the + other cofactors shift by the same amount (see above). It must be one of + ``cofactor_met_names`` and in the biomass pseudoreaction. ``None`` + sets every cofactor to ``±value``. ngam_rxn, ngam_value Optional NGAM update. If both are provided, the reaction's bounds are set to ``(ngam_value, ngam_value)`` (equality @@ -59,7 +74,8 @@ def set_gam( If ``biomass_rxn``, or a given ``ngam_rxn``, is not in the model. Both reactions are looked up before anything is changed. ValueError - If ``ngam_value`` is given without an ``ngam_rxn``. + If ``ngam_value`` is given without an ``ngam_rxn``, or ``keep_offsets_of`` + names no cofactor of the biomass pseudoreaction. """ if ngam_value is not None and not ngam_rxn: raise ValueError("ngam_value needs an ngam_rxn: the NGAM reaction to set is not given") @@ -68,14 +84,44 @@ def set_gam( cofactor_set = set(cofactor_met_names) targets: dict[cobra.Metabolite, float] = {} + shift = 0.0 + if keep_offsets_of is not None: + shift = float(value) - _gam_coefficient(rxn, keep_offsets_of, cofactor_set) for met, coef in rxn.metabolites.items(): if met.name in cofactor_set: - target = (1 if coef > 0 else -1) * float(value) - targets[met] = target - if targets: + sign = 1 if coef > 0 else -1 + targets[met] = sign * shift if keep_offsets_of is not None else sign * float(value) + if targets and keep_offsets_of is not None: + rxn.add_metabolites(targets, combine=True) # adds the shift, which a context reverts + elif targets: rxn.add_metabolites(targets, combine=False) # replaces the coefficient, so it is exactly target if ngam is not None and ngam_value is not None: ngam.bounds = (float(ngam_value), float(ngam_value)) return model + + +def get_gam(model: cobra.Model, *, biomass_rxn: str, atp_met_name: str = "ATP") -> float: + """GAM of the model: the size of the ATP coefficient of the biomass pseudoreaction. + + Raises + ------ + KeyError + If ``biomass_rxn`` is not in the model. + ValueError + If the pseudoreaction has no metabolite named ``atp_met_name``. + """ + rxn = model.reactions.get_by_id(biomass_rxn) + return _gam_coefficient(rxn, atp_met_name, {atp_met_name}) + + +def _gam_coefficient(rxn: cobra.Reaction, met_name: str, cofactor_names: set[str]) -> float: + if met_name not in cofactor_names: + raise ValueError(f"keep_offsets_of {met_name!r} is not one of the cofactor names") + found = [abs(coef) for met, coef in rxn.metabolites.items() if met.name == met_name] + if len(found) != 1: + raise ValueError( + f"{rxn.id} has {len(found)} metabolites named {met_name!r}; the GAM term needs exactly one" + ) + return found[0] diff --git a/src/raven_toolbox/biomass/maintenance.py b/src/raven_toolbox/biomass/maintenance.py new file mode 100644 index 0000000..8bcb4e3 --- /dev/null +++ b/src/raven_toolbox/biomass/maintenance.py @@ -0,0 +1,357 @@ +"""Fit GAM and NGAM to measured substrate uptake and maintenance coefficients. + +GAM (the ATP hydrolysed per gram of biomass formed) and NGAM (the ATP hydrolysed per hour +and gram of biomass) are fixed by different data, so they are fitted one after the other: + +* NGAM from the maintenance coefficient measured in retentostats, or in chemostats at very + low growth rates. The substrate uptake that a model needs at zero growth is proportional + to NGAM, so :func:`ngam_from_maintenance_coefficient` needs one solve. +* GAM from the substrate uptake of chemostats at the measured growth rates, with NGAM held + at its value (:func:`fit_gam`). The model uptake is computed once on a grid of GAM values + (:func:`uptake_grid`); the loss, the best GAM and the resampling of studies + (:func:`bootstrap_gam`) then work on that grid and need no further solves. + +The chemostat data come from many studies. The loss weights every study equally +(:func:`study_weights`: a unit counts ``1 / sqrt`` of the number of units of its study, and +units from a less reliable origin count less) and is the Huber loss of the log ratio of model +to measured uptake (:func:`huber_loss`, :func:`log_ratio`), so that one study that +disagrees with the others does not move the result. + +:func:`fit_maintenance` runs the steps for one data set. A model that has several regimes, +for example aerobic and anaerobic growth, is fitted once per regime with the model as it is +used in that regime. + +The ids of the reactions and the names of the metabolites are given in a +:class:`MaintenanceConfig`. A step that depends on the data, such as setting the protein +content of the biomass of a chemostat, is a function in ``condition_setup``. +""" +from __future__ import annotations + +from collections.abc import Callable, Iterable, Sequence +from dataclasses import dataclass, field + +import cobra +import numpy as np +import pandas as pd + +from raven_toolbox.biomass.gam import get_gam, set_gam + +__all__ = [ + "DEFAULT_GAMS", + "HUBER_DELTA", + "INFEASIBLE_LOG_RATIO", + "MaintenanceConfig", + "MaintenanceFit", + "bootstrap_gam", + "fit_gam", + "fit_maintenance", + "huber_loss", + "log_ratio", + "min_uptake", + "ngam_from_maintenance_coefficient", + "study_weights", + "uptake_grid", +] + +#: Transition of the Huber loss, as a log ratio: residuals below 0.15 (about 15%) count +#: quadratically, larger ones linearly. +HUBER_DELTA = 0.15 +#: Log ratio assigned to a growth rate that the model cannot sustain (about a factor 2). +INFEASIBLE_LOG_RATIO = 0.7 +#: GAM values (mmol/gDW) at which the model uptake is computed. +DEFAULT_GAMS = np.arange(25.0, 70.01, 1.25) + + +@dataclass(frozen=True) +class MaintenanceConfig: + """Reactions and metabolites that carry GAM and NGAM in a model. + + Attributes + ---------- + biomass_rxn: + Biomass pseudoreaction that holds the GAM coefficients. + ngam_rxn: + Maintenance reaction whose fixed flux is NGAM. + growth_rxn: + Reaction whose flux is the growth rate, fixed to the dilution rate of a chemostat. + Often the same as ``biomass_rxn``, or a reaction that drains the biomass. + substrate_exchange: + Exchange reaction of the limiting substrate. Its uptake is a negative flux. + cofactor_met_names: + Names of the metabolites of the GAM term in the biomass pseudoreaction, such as ATP, + ADP, H2O, H+ and phosphate. + atp_met_name: + Name of the ATP metabolite, whose coefficient is the GAM. + """ + + biomass_rxn: str + ngam_rxn: str + growth_rxn: str + substrate_exchange: str + cofactor_met_names: tuple[str, ...] + atp_met_name: str = "ATP" + + +@dataclass +class MaintenanceFit: + """The values fitted to one data set. + + Attributes + ---------- + ngam, gam: + Fitted NGAM and GAM. + gam_p16, gam_p84: + 16th and 84th percentile of the best GAM when the studies are resampled. + units: + Number of data rows. + studies: + Number of studies. + median_error: + Median relative difference between model and measured uptake at the fitted GAM. + """ + + ngam: float + gam: float + gam_p16: float + gam_p84: float + units: int + studies: int + median_error: float + + +def huber_loss(residual: np.ndarray, delta: float = HUBER_DELTA) -> np.ndarray: + """Huber loss: ``r**2 / 2`` for ``|r| <= delta``, ``delta * (|r| - delta / 2)`` above.""" + r = np.asarray(residual, dtype=float) + a = np.abs(r) + return np.where(a <= delta, 0.5 * r * r, delta * (a - 0.5 * delta)) + + +def log_ratio( + simulated: np.ndarray, measured: np.ndarray, infeasible: float = INFEASIBLE_LOG_RATIO, +) -> np.ndarray: + """Natural log of ``simulated / measured``, row-wise broadcast over ``simulated``. + + A value that is not finite, such as the NaN of a growth rate the model cannot sustain, + gives ``infeasible``. + """ + with np.errstate(all="ignore"): + r = np.log(np.asarray(simulated, dtype=float) / np.asarray(measured, dtype=float)[None, :]) + return np.where(np.isfinite(r), r, infeasible) + + +def study_weights( + study: Iterable, *, discounted: Iterable[bool] | None = None, discount: float = 0.5, +) -> np.ndarray: + """Weight of every data row: ``1 / sqrt(rows of its study)``, times ``discount`` where + ``discounted`` is True (for example values read from a figure).""" + counts = pd.Series(list(study)).groupby(list(study)).transform("size").to_numpy(dtype=float) + weight = 1.0 / np.sqrt(counts) + if discounted is not None: + weight = weight * np.where(np.asarray(list(discounted), dtype=bool), discount, 1.0) + return weight + + +def min_uptake( + model: cobra.Model, + growth: float, + config: MaintenanceConfig, + *, + setup: Callable[[cobra.Model], None] | None = None, +) -> float: + """Smallest substrate uptake (a positive number) at a fixed growth rate. + + The model is changed only inside ``with model:``, so it is as before after the call. + ``setup`` is called on the model before the solve. The result is NaN when the growth + rate cannot be reached. + """ + with model: + if setup is not None: + setup(model) + substrate = model.reactions.get_by_id(config.substrate_exchange) + substrate.lower_bound = -1000.0 + model.reactions.get_by_id(config.growth_rxn).bounds = (growth, growth) + model.objective = {substrate: 1.0} # an uptake is a negative flux + value = model.slim_optimize(error_value=np.nan) + return -value + + +def uptake_grid( + model: cobra.Model, + data: pd.DataFrame, + gams: Sequence[float], + ngam: float, + config: MaintenanceConfig, + *, + growth_col: str = "growth_rate", + condition_setup: Callable[[cobra.Model, pd.Series], None] | None = None, +) -> np.ndarray: + """Model uptake for every GAM in ``gams`` (rows) and data row (columns). + + ``condition_setup(model, row)`` is called on the model for each row, for example to set + the protein content that was measured in that chemostat. + """ + out = np.full((len(gams), len(data)), np.nan) + growth = data[growth_col].to_numpy(dtype=float) + rows = [data.iloc[j] for j in range(len(data))] + for i, gam in enumerate(gams): + with model: + _set_maintenance(model, float(gam), ngam, config) + for j, row in enumerate(rows): + setup = None if condition_setup is None else (lambda m, r=row: condition_setup(m, r)) + out[i, j] = min_uptake(model, growth[j], config, setup=setup) + return out + + +def ngam_from_maintenance_coefficient( + model: cobra.Model, + ms_substrate: float, + config: MaintenanceConfig, + *, + gam: float | None = None, +) -> float: + """NGAM at which the model needs ``ms_substrate`` (mmol/gDW/h) at zero growth. + + The uptake at zero growth is proportional to NGAM, so one solve is enough. The ATP yield + of the substrate comes from the model. ``gam`` is set first when given. + """ + with model: + _set_maintenance(model, gam, 1.0, config) + per_unit = min_uptake(model, 0.0, config) + return float(ms_substrate / per_unit) + + +def fit_gam( + model: cobra.Model, + data: pd.DataFrame, + ngam: float, + config: MaintenanceConfig, + *, + gams: Sequence[float] | None = None, + growth_col: str = "growth_rate", + uptake_col: str = "uptake", + weights: np.ndarray | None = None, + condition_setup: Callable[[cobra.Model, pd.Series], None] | None = None, +) -> tuple[float, np.ndarray, np.ndarray]: + """Best GAM for the rows of ``data`` at a given NGAM. + + ``weights`` weight the rows (default: all 1; see :func:`study_weights`). Returns the GAM, + the GAM grid, and the matrix of log ratios (GAM by row of ``data``), which + :func:`bootstrap_gam` resamples without new solves. + """ + gams = np.asarray(DEFAULT_GAMS if gams is None else gams, dtype=float) + measured = data[uptake_col].to_numpy(dtype=float) + ratios = log_ratio( + uptake_grid(model, data, gams, ngam, config, growth_col=growth_col, + condition_setup=condition_setup), + measured, + ) + w = np.ones(len(data)) if weights is None else np.asarray(weights, dtype=float) + return _best_gam(gams, ratios, w), gams, ratios + + +def bootstrap_gam( + gams: Sequence[float], + ratios: np.ndarray, + study: Iterable, + weights: np.ndarray, + *, + n: int = 400, + seed: int = 0, + percentiles: tuple[float, float] = (16.0, 84.0), +) -> tuple[float, float]: + """Percentiles of the best GAM when the studies are resampled with replacement. + + ``study`` gives the study of every column of ``ratios``; a resample draws as many studies + as there are, and a row counts as often as its study was drawn. + """ + rng = np.random.default_rng(seed) + gams = np.asarray(gams, dtype=float) + labels = np.asarray(list(study)) + names = sorted(set(labels)) + best = [] + for _ in range(n): + drawn = rng.choice(len(names), len(names)) + count = np.zeros(len(labels)) + for k in drawn: + count[labels == names[k]] += 1 + w = weights * count + if w.sum() == 0: + continue + best.append(_best_gam(gams, ratios, w)) + lo, hi = np.percentile(best, percentiles) + return float(lo), float(hi) + + +def fit_maintenance( + model: cobra.Model, + data: pd.DataFrame, + ms_substrate: float, + config: MaintenanceConfig, + *, + study_col: str = "study", + discounted: Iterable[bool] | None = None, + discount: float = 0.5, + growth_col: str = "growth_rate", + uptake_col: str = "uptake", + gams: Sequence[float] | None = None, + bootstrap: int = 400, + seed: int = 0, + condition_setup: Callable[[cobra.Model, pd.Series], None] | None = None, +) -> MaintenanceFit: + """NGAM from the maintenance coefficient and GAM from the chemostats of ``data``. + + Parameters + ---------- + model: + Model as it is used for the growth regime of ``data``. It is not changed. + data: + One row per chemostat, with the growth rate (``growth_col``), the measured substrate + uptake (``uptake_col``) and the study (``study_col``). + ms_substrate: + Maintenance coefficient of the substrate, mmol/gDW/h, from retentostats. + config: + The reactions and metabolites of GAM and NGAM. + discounted, discount: + Rows that count ``discount`` times as much in the weights, for example values read + from figures. + gams: + GAM values of the grid (default :data:`DEFAULT_GAMS`). + bootstrap, seed: + Number of resamples of the studies and the seed. + condition_setup: + Function ``(model, row)`` that sets the model for the conditions of one chemostat. + """ + data = data.reset_index(drop=True) + ngam = ngam_from_maintenance_coefficient(model, ms_substrate, config) + study = data[study_col].to_numpy() + weights = study_weights(study, discounted=discounted, discount=discount) + gam, grid, ratios = fit_gam( + model, data, ngam, config, gams=gams, growth_col=growth_col, + uptake_col=uptake_col, weights=weights, condition_setup=condition_setup, + ) + lo, hi = bootstrap_gam(grid, ratios, study, weights, n=bootstrap, seed=seed) + best = ratios[int(np.argmin(np.abs(grid - gam)))] + return MaintenanceFit( + ngam=ngam, gam=gam, gam_p16=lo, gam_p84=hi, + units=len(data), studies=len(set(study)), + median_error=float(np.median(np.abs(np.exp(best) - 1.0))), + ) + + +def _best_gam(gams: np.ndarray, ratios: np.ndarray, weights: np.ndarray) -> float: + loss = (huber_loss(ratios) * weights[None, :]).sum(axis=1) / weights.sum() + return float(gams[int(np.argmin(loss))]) + + +def _set_maintenance( + model: cobra.Model, gam: float | None, ngam: float, config: MaintenanceConfig, +) -> None: + """Set GAM (when given) and NGAM in place, keeping the offsets between the GAM terms.""" + if gam is None: + gam = get_gam(model, biomass_rxn=config.biomass_rxn, atp_met_name=config.atp_met_name) + set_gam( + model, gam, biomass_rxn=config.biomass_rxn, + cofactor_met_names=config.cofactor_met_names, + ngam_rxn=config.ngam_rxn, ngam_value=ngam, + keep_offsets_of=config.atp_met_name, + ) diff --git a/tests/test_biomass_maintenance.py b/tests/test_biomass_maintenance.py new file mode 100644 index 0000000..9f6ba19 --- /dev/null +++ b/tests/test_biomass_maintenance.py @@ -0,0 +1,292 @@ +"""Tests for the GAM and NGAM fit (``raven_toolbox.biomass.maintenance``). + +The toy model makes the uptake analytic. Glucose gives 2 ATP; the biomass reaction takes +5 glucose and ``GAM`` ATP per unit of biomass; the maintenance reaction takes ``NGAM`` ATP. +The glucose uptake at growth ``mu`` is therefore ``5 * mu + (GAM * mu + NGAM) / 2``. +""" +import cobra +import numpy as np +import pandas as pd +import pytest + +from raven_toolbox.biomass import ( + MaintenanceConfig, + bootstrap_gam, + fit_gam, + fit_maintenance, + get_gam, + huber_loss, + log_ratio, + min_uptake, + ngam_from_maintenance_coefficient, + set_gam, + study_weights, + uptake_grid, +) + +NAMES = ("ATP", "ADP", "H2O", "H+", "phosphate") +CONFIG = MaintenanceConfig( + biomass_rxn="BIO", ngam_rxn="NGAM", growth_rxn="GROWTH", + substrate_exchange="EX_glc", cofactor_met_names=NAMES, +) +PROTON_OFFSET = 0.3 # balances the charge of the other biomass terms + + +def _toy(gam=30.0, ngam=1.0): + model = cobra.Model("toy") + mets = { + i: cobra.Metabolite(i, name=n, compartment="c") + for i, n in [("glc", "glucose"), ("atp", "ATP"), ("adp", "ADP"), ("h2o", "H2O"), + ("h", "H+"), ("pi", "phosphate"), ("bio", "biomass")] + } + m = mets + + def rxn(rid, stoich, lb=0.0, ub=1000.0): + r = cobra.Reaction(rid) + r.add_metabolites({m[k]: v for k, v in stoich.items()}) + r.bounds = (lb, ub) + return r + + model.add_reactions([ + rxn("EX_glc", {"glc": -1}, -1000, 1000), + rxn("ATPPROD", {"glc": -1, "adp": -2, "pi": -2, "h": -2, "atp": 2, "h2o": 2}), + rxn("BIO", {"glc": -5, "atp": -gam, "h2o": -gam, "adp": gam, "pi": gam, + "h": gam + PROTON_OFFSET, "bio": 1}), + rxn("GROWTH", {"bio": -1}), + rxn("NGAM", {"atp": -1, "h2o": -1, "adp": 1, "pi": 1, "h": 1}, ngam, ngam), + rxn("EX_h2o", {"h2o": -1}, -1000, 1000), + rxn("EX_h", {"h": -1}, -1000, 1000), + rxn("EX_pi", {"pi": -1}, -1000, 1000), + ]) + model.objective = "GROWTH" + return model + + +def _uptake(mu, gam, ngam): + return 5 * mu + (gam * mu + ngam) / 2 + + +# --- set_gam and get_gam --------------------------------------------------------------- + + +def test_get_gam_is_the_atp_coefficient(): + assert get_gam(_toy(gam=30.0), biomass_rxn="BIO") == 30.0 + + +def test_get_gam_needs_one_atp_in_the_reaction(): + with pytest.raises(ValueError, match="needs exactly one"): + get_gam(_toy(), biomass_rxn="BIO", atp_met_name="nothing") + + +def test_keep_offsets_shifts_every_term_and_keeps_the_proton_offset(): + model = _toy(gam=30.0) + set_gam(model, 45.0, biomass_rxn="BIO", cofactor_met_names=NAMES, keep_offsets_of="ATP") + coef = {met.name: c for met, c in model.reactions.BIO.metabolites.items()} + assert coef["ATP"] == -45.0 and coef["H2O"] == -45.0 + assert coef["ADP"] == 45.0 and coef["phosphate"] == 45.0 + assert coef["H+"] == pytest.approx(45.0 + PROTON_OFFSET) + + +def test_without_keep_offsets_every_term_is_set_to_the_value(): + model = _toy(gam=30.0) + set_gam(model, 45.0, biomass_rxn="BIO", cofactor_met_names=NAMES) + coef = {met.name: c for met, c in model.reactions.BIO.metabolites.items()} + assert coef["H+"] == 45.0 + + +def test_keep_offsets_reverts_in_a_context(): + model = _toy(gam=30.0) + with model: + set_gam(model, 45.0, biomass_rxn="BIO", cofactor_met_names=NAMES, + ngam_rxn="NGAM", ngam_value=2.0, keep_offsets_of="ATP") + assert get_gam(model, biomass_rxn="BIO") == 45.0 + assert get_gam(model, biomass_rxn="BIO") == 30.0 + assert model.reactions.NGAM.bounds == (1.0, 1.0) + + +def test_keep_offsets_of_must_be_a_cofactor(): + with pytest.raises(ValueError, match="not one of the cofactor names"): + set_gam(_toy(), 45.0, biomass_rxn="BIO", cofactor_met_names=("ADP",), keep_offsets_of="ATP") + + +# --- loss, ratios, weights ---------------------------------------------------------------- + + +def test_huber_loss_is_quadratic_then_linear(): + r = np.array([0.0, 0.1, -0.1, 0.45]) + assert huber_loss(r, delta=0.15) == pytest.approx( + [0.0, 0.005, 0.005, 0.15 * (0.45 - 0.075)]) + + +def test_log_ratio_broadcasts_and_penalises_infeasible_rows(): + grid = np.array([[2.0, np.nan], [1.0, 4.0]]) + out = log_ratio(grid, np.array([1.0, 2.0]), infeasible=0.7) + assert out == pytest.approx(np.array([[np.log(2.0), 0.7], [0.0, np.log(2.0)]])) + + +def test_study_weights_count_studies_equally_and_discount(): + w = study_weights(["a"] * 4 + ["b"] * 4 + ["c"], + discounted=[False] * 4 + [True] * 4 + [False]) + assert w[:4] == pytest.approx(0.5) + assert w[4:8] == pytest.approx(0.25) + assert w[8] == pytest.approx(1.0) + + +def test_study_weights_without_discount(): + assert study_weights(["a", "a", "b"]) == pytest.approx([2 ** -0.5, 2 ** -0.5, 1.0]) + + +# --- simulation ------------------------------------------------------------------------------ + + +def test_min_uptake_matches_the_analytic_value_and_leaves_the_model_as_it_was(): + model = _toy(gam=30.0, ngam=1.0) + assert min_uptake(model, 0.1, CONFIG) == pytest.approx(_uptake(0.1, 30.0, 1.0)) + assert model.reactions.GROWTH.bounds == (0.0, 1000.0) + assert model.reactions.EX_glc.lower_bound == -1000.0 + assert model.objective.expression.has(model.reactions.GROWTH.forward_variable) + + +def test_min_uptake_is_nan_when_the_growth_rate_cannot_be_reached(): + model = _toy() + model.reactions.ATPPROD.upper_bound = 0.1 # too little ATP for the growth rate + assert np.isnan(min_uptake(model, 0.8, CONFIG)) + + +def test_min_uptake_runs_the_setup_inside_the_context(): + model = _toy(gam=30.0) + + def setup(m): + m.reactions.EX_glc.upper_bound = 0.0 + m.reactions.NGAM.bounds = (3.0, 3.0) + + assert min_uptake(model, 0.0, CONFIG, setup=setup) == pytest.approx(1.5) + assert model.reactions.NGAM.bounds == (1.0, 1.0) + + +def test_uptake_grid_covers_every_gam_and_row(): + model = _toy() + data = pd.DataFrame({"growth_rate": [0.05, 0.1, 0.2]}) + grid = uptake_grid(model, data, [20.0, 40.0], 0.8, CONFIG) + expected = [[_uptake(mu, g, 0.8) for mu in data["growth_rate"]] for g in (20.0, 40.0)] + assert grid == pytest.approx(np.array(expected)) + assert get_gam(model, biomass_rxn="BIO") == 30.0 + + +def test_uptake_grid_calls_the_condition_setup_for_each_row(): + model = _toy() + data = pd.DataFrame({"growth_rate": [0.1, 0.1], "extra": [0.0, 10.0]}) + + def setup(m, row): + m.reactions.EX_glc.upper_bound = row["extra"] # not binding: only checks the call + + seen = [] + + def spy(m, row): + seen.append(row["extra"]) + setup(m, row) + + uptake_grid(model, data, [30.0], 1.0, CONFIG, condition_setup=spy) + assert seen == [0.0, 10.0] + + +def test_ngam_from_maintenance_coefficient_inverts_the_uptake_at_zero_growth(): + model = _toy(gam=30.0, ngam=1.0) + ngam = ngam_from_maintenance_coefficient(model, 0.4, CONFIG) + assert ngam == pytest.approx(0.8) # uptake at zero growth is NGAM / 2 + assert model.reactions.NGAM.bounds == (1.0, 1.0) + + +def test_ngam_from_maintenance_coefficient_with_a_given_gam_keeps_offsets(): + model = _toy(gam=30.0) + assert ngam_from_maintenance_coefficient(model, 0.4, CONFIG, gam=50.0) == pytest.approx(0.8) + assert get_gam(model, biomass_rxn="BIO") == 30.0 + + +# --- fit ----------------------------------------------------------------------------------------- + +TRUE_GAM, TRUE_NGAM = 40.0, 0.8 + + +def _chemostats(noise=0.0, seed=1): + rng = np.random.default_rng(seed) + rows = [] + for study in "abcdef": + for mu in (0.05, 0.1, 0.2, 0.3): + up = _uptake(mu, TRUE_GAM, TRUE_NGAM) * (1 + noise * rng.standard_normal()) + rows.append({"study": study, "growth_rate": mu, "uptake": up}) + return pd.DataFrame(rows) + + +def test_fit_gam_recovers_the_gam_the_data_were_made_with(): + model = _toy(gam=30.0) + data = _chemostats() + gam, grid, ratios = fit_gam(model, data, TRUE_NGAM, CONFIG, gams=np.arange(30.0, 50.1, 2.5)) + assert gam == 40.0 + assert ratios.shape == (len(grid), len(data)) + assert ratios[list(grid).index(40.0)] == pytest.approx(0.0, abs=1e-9) + + +def test_fit_gam_one_outlier_study_moves_the_result_by_less_than_a_grid_step(): + """A study that reads three times the uptake of the others, with a quadratic loss, + would pull the best GAM far above 40; the Huber loss keeps it within one step.""" + model = _toy(gam=30.0) + data = _chemostats() + data.loc[data["study"] == "a", "uptake"] *= 3.0 + weights = study_weights(data["study"]) + gam, _, _ = fit_gam(model, data, TRUE_NGAM, CONFIG, gams=np.arange(30.0, 50.1, 2.5), + weights=weights) + assert abs(gam - 40.0) <= 2.5 + + +def test_bootstrap_gam_is_exact_without_noise_and_wider_with_a_dissenting_study(): + gams = np.arange(30.0, 50.1, 2.5) + model = _toy(gam=30.0) + data = _chemostats() + weights = study_weights(data["study"]) + _, grid, ratios = fit_gam(model, data, TRUE_NGAM, CONFIG, gams=gams, weights=weights) + assert bootstrap_gam(grid, ratios, data["study"], weights, n=50) == (40.0, 40.0) + + data.loc[data["study"] == "a", "uptake"] *= 1.5 + _, grid, ratios = fit_gam(model, data, TRUE_NGAM, CONFIG, gams=gams, weights=weights) + lo, hi = bootstrap_gam(grid, ratios, data["study"], weights, n=200) + assert lo <= 40.0 <= hi and hi > lo + + +def test_bootstrap_gam_is_reproducible_by_seed(): + gams = np.array([30.0, 40.0, 50.0]) + rng = np.random.default_rng(0) + ratios = rng.normal(size=(3, 12)) + study = list("aabbccddeeff") + w = np.ones(12) + assert bootstrap_gam(gams, ratios, study, w, seed=3) == bootstrap_gam(gams, ratios, study, w, seed=3) + + +def test_fit_maintenance_returns_the_fit_and_leaves_the_model_alone(): + model = _toy(gam=30.0) + data = _chemostats() + fit = fit_maintenance(model, data, TRUE_NGAM / 2, CONFIG, + gams=np.arange(30.0, 50.1, 2.5), bootstrap=30) + assert fit.ngam == pytest.approx(TRUE_NGAM) + assert fit.gam == 40.0 and fit.gam_p16 == 40.0 and fit.gam_p84 == 40.0 + assert fit.units == 24 and fit.studies == 6 + assert fit.median_error == pytest.approx(0.0, abs=1e-9) + assert get_gam(model, biomass_rxn="BIO") == 30.0 + assert model.reactions.NGAM.bounds == (1.0, 1.0) + + +def test_fit_maintenance_discounts_rows_and_sets_up_conditions(): + model = _toy(gam=30.0) + data = _chemostats() + data["note"] = np.where(data["study"] == "a", "figure", "table") + calls = [] + + def setup(m, row): + calls.append(row["study"]) + + fit = fit_maintenance( + model, data, TRUE_NGAM / 2, CONFIG, gams=np.arange(30.0, 50.1, 2.5), bootstrap=5, + discounted=data["note"] == "figure", condition_setup=setup, + ) + assert fit.gam == 40.0 + assert len(calls) == 24 * 9 # nine GAM values From 4cce72ff7e58faedf4e64967254f1e57f61c0f87 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Fri, 20 Nov 2026 23:07:35 +0100 Subject: [PATCH 5/6] fix: leave a condition with weight 0 out of the simulation and the distance simulate_conditions does not match or simulate a condition with weight 0 and marks it skipped. dataset_rmse gives it the error NaN and leaves it out of the mean, and experiment_distance leaves out a dataset without a condition that has a weight other than 0. The distance is NaN if every condition has weight 0. --- CHANGELOG.md | 2 +- src/raven_toolbox/experiments/conditions.py | 44 +++++-- src/raven_toolbox/experiments/data.py | 2 +- src/raven_toolbox/experiments/distance.py | 46 ++++--- src/raven_toolbox/experiments/flux_data.py | 10 +- tests/test_experiments_conditions.py | 138 ++++++++++++++++++++ 6 files changed, 211 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84ef756..5b033ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ ### Experiments -* New: the `experiments` package scores a model against measured growth rates and exchange fluxes. `load_experiment_data` reads `fluxData.tsv`, `maxGrowth.tsv` and `zeroFlux.tsv` from a folder, `load_flux_data` parses one flux file into a `FluxData` (with a per-condition `weight`), `simulate_conditions` fixes the uptake of each condition and solves the model, `compute_carbon_weights` gives each exchange the carbon count of its metabolite, `experiment_distance` combines the carbon-weighted RMSE of flux and max-growth conditions into one value, and `apply_flux_data_constraints` sets the exchange bounds of a model from one condition (loose, or a percentage band). The organism-specific steps, an anaerobic switch and a change of the protein content, and the solver call are passed as functions. +* New: the `experiments` package scores a model against measured growth rates and exchange fluxes. `load_experiment_data` reads `fluxData.tsv`, `maxGrowth.tsv` and `zeroFlux.tsv` from a folder, `load_flux_data` parses one flux file into a `FluxData` (with a per-condition `weight`), `simulate_conditions` fixes the uptake of each condition and solves the model, `compute_carbon_weights` gives each exchange the carbon count of its metabolite, `experiment_distance` combines the carbon-weighted RMSE of flux and max-growth conditions into one value, and `apply_flux_data_constraints` sets the exchange bounds of a model from one condition (loose, or a percentage band). The organism-specific steps, an anaerobic switch and a change of the protein content, and the solver call are passed as functions. A condition with weight 0 is left out of the simulation and the distance. ### Solver diff --git a/src/raven_toolbox/experiments/conditions.py b/src/raven_toolbox/experiments/conditions.py index cf73e9f..dfb5cab 100644 --- a/src/raven_toolbox/experiments/conditions.py +++ b/src/raven_toolbox/experiments/conditions.py @@ -29,12 +29,17 @@ class ConditionSimResult: means ``growth``/``exch_fluxes``/``block_fluxes`` are meaningless; :mod:`.distance` applies the fixed infeasibility penalty for that row instead of reading them. + + ``skipped=True`` marks a condition with weight 0, which is not + simulated; it is also ``feasible=False``, and :mod:`.distance` leaves + it out of the distance. """ feasible: bool growth: float = float("nan") exch_fluxes: dict[str, float] = field(default_factory=dict) block_fluxes: dict[str, float] = field(default_factory=dict) + skipped: bool = False def simulate_conditions( @@ -52,8 +57,12 @@ def simulate_conditions( The steps are: + 0. A condition with ``flux_data.weight`` exactly 0 is left out: it is + not matched to an exchange, not simulated, and its result is + ``ConditionSimResult(feasible=False, skipped=True)``. Its name need + not match any exchange metabolite. 1. Every exchange reaction that appears as a *condition name* - anywhere in ``flux_data.conds`` (matched by name against + of a condition with a weight other than 0 (matched by name against ``flux_data.exch_mets``) is blocked (``lower_bound = 0``) up front, across the board -- not just the reaction for the row being solved -- so one row's carbon source can't leak flux into @@ -112,14 +121,15 @@ def simulate_conditions( Raises ------ ValueError - If any ``flux_data.conds`` entry can't be matched by name to - an exchange metabolite in ``flux_data.exch_mets``. + If the name of a condition with a weight other than 0 can't be + matched to an exchange metabolite in ``flux_data.exch_mets``. """ if optimize is None: optimize = _default_optimize - cond_to_exch_idx = _match_conditions_to_exch(flux_data) + active = _active_conditions(flux_data) + cond_to_exch_idx = _match_conditions_to_exch(flux_data, active) all_condition_rxn_ids = sorted( - {flux_data.exch_rxn_ids[j] for j in set(cond_to_exch_idx)} + {flux_data.exch_rxn_ids[j] for j in set(cond_to_exch_idx.values())} ) o2_col = _find_col(flux_data.exch_mets, "oxygen") measured_cols = [ @@ -127,7 +137,11 @@ def simulate_conditions( ] if constrain else [] results: list[ConditionSimResult] = [] - for i, exch_idx in enumerate(cond_to_exch_idx): + for i in range(len(flux_data.conds)): + if i not in cond_to_exch_idx: + results.append(ConditionSimResult(feasible=False, skipped=True)) + continue + exch_idx = cond_to_exch_idx[i] carbon_rxn_id = flux_data.exch_rxn_ids[exch_idx] with model: for rxn_id in all_condition_rxn_ids: @@ -185,14 +199,24 @@ def _default_optimize(model: "cobra.Model") -> "cobra.Solution": return model.optimize() -def _match_conditions_to_exch(flux_data: "FluxData") -> list[int]: - """Column index into ``exch_mets``/``exch_rxn_ids`` for each +def _active_conditions(flux_data: "FluxData") -> list[int]: + """Indices of the conditions with a weight other than 0.""" + if flux_data.weight is None: + return list(range(len(flux_data.conds))) + return [i for i in range(len(flux_data.conds)) if flux_data.weight[i] != 0] + + +def _match_conditions_to_exch( + flux_data: "FluxData", active: list[int], +) -> dict[int, int]: + """Column index into ``exch_mets``/``exch_rxn_ids`` for each active condition, matched by name. Raises ``ValueError`` when a condition cannot be matched.""" met_to_idx: dict[str, int] = {} for j, met in enumerate(flux_data.exch_mets): met_to_idx.setdefault(met, j) - missing = sorted({c for c in flux_data.conds if c not in met_to_idx}) + missing = sorted({flux_data.conds[i] for i in active + if flux_data.conds[i] not in met_to_idx}) if missing: raise ValueError( 'Carbon source(s) "' @@ -200,7 +224,7 @@ def _match_conditions_to_exch(flux_data: "FluxData") -> list[int]: + '" in the provided fluxData or maxGrowth cannot be matched ' "by name with an exchange reaction." ) - return [met_to_idx[c] for c in flux_data.conds] + return {i: met_to_idx[flux_data.conds[i]] for i in active} def _find_col(exch_mets: list[str], name: str) -> Optional[int]: diff --git a/src/raven_toolbox/experiments/data.py b/src/raven_toolbox/experiments/data.py index bca9578..2484f1e 100644 --- a/src/raven_toolbox/experiments/data.py +++ b/src/raven_toolbox/experiments/data.py @@ -5,7 +5,7 @@ ``Condition Ptot grRate () ... weight source``, so both use :func:`~raven_toolbox.experiments.flux_data.load_flux_data`; only ``zeroFlux.tsv`` (a single ``Rxns`` id column) needs a loader of its own. A ``weight`` of 0 leaves a -condition out of the distance to the measurements. +condition out of the simulation and of the distance to the measurements. All three files are optional: a missing file yields ``None`` (or an empty list for ``zero_flux``), never an exception. diff --git a/src/raven_toolbox/experiments/distance.py b/src/raven_toolbox/experiments/distance.py index 5594471..7c00de1 100644 --- a/src/raven_toolbox/experiments/distance.py +++ b/src/raven_toolbox/experiments/distance.py @@ -97,6 +97,9 @@ def dataset_rmse( ) -> tuple[float, np.ndarray]: """RMSE (mean, per-condition array) for one dataset. + A condition with ``flux_data.weight`` exactly 0 is left out: its entry in + the per-condition array is ``NaN`` and it is not part of the mean. + Parameters ---------- flux_data @@ -105,7 +108,8 @@ def dataset_rmse( sims One :class:`~.conditions.ConditionSimResult` per ``flux_data.conds`` row, e.g. from - :func:`~.conditions.simulate_conditions`. + :func:`~.conditions.simulate_conditions`. The entries of the + conditions with weight 0 are not read. constrain Must match the value passed to :func:`~.conditions.simulate_conditions` for these @@ -126,10 +130,11 @@ def dataset_rmse( ------- rmse : float Weighted (by ``flux_data.weight``, if present) - mean of the per-condition RMSEs. + mean of the per-condition RMSEs of the conditions with a weight + other than 0. ``NaN`` if every condition has weight 0. rmse_list : numpy.ndarray The per-condition RMSEs (post-penalty, pre-weighting), shape - ``(len(flux_data.conds),)``. + ``(len(flux_data.conds),)``; ``NaN`` for a condition with weight 0. """ n = len(flux_data.conds) if len(sims) != n: @@ -139,8 +144,16 @@ def dataset_rmse( ) rxn_to_col = {r: j for j, r in enumerate(flux_data.exch_rxn_ids)} + if flux_data.weight is None: + active = np.ones(n, dtype=bool) + else: + active = np.asarray(flux_data.weight) != 0 + rmse_list = np.empty(n, dtype=float) for i, sim in enumerate(sims): + if not active[i]: + rmse_list[i] = np.nan + continue if not sim.feasible: rmse_list[i] = penalty continue @@ -170,10 +183,12 @@ def dataset_rmse( diff = np.asarray(measured, dtype=float) - np.asarray(simulated, dtype=float) rmse_list[i] = float(np.sqrt(np.mean(diff ** 2))) + if not active.any(): + return float("nan"), rmse_list weighted = rmse_list if flux_data.weight is not None: weighted = rmse_list * flux_data.weight - return float(np.mean(weighted)), rmse_list + return float(np.mean(weighted[active])), rmse_list def experiment_distance( @@ -191,24 +206,23 @@ def experiment_distance( ``flux_sims`` must be given (non-``None``) iff ``experiment_data.flux_data`` is not ``None``, and likewise for ``max_growth_sims`` / ``experiment_data.max_growth``. A dataset absent from ``experiment_data`` doesn't - contribute to the combined RMSE. + contribute to the combined RMSE, and neither does a dataset in which + every condition has weight 0. Parameters ---------- max_growth_weight Relative weight of the max-growth dataset against the flux dataset, giving ``(rmse_flux + w * rmse_max_growth) / (w + 1)``. - At 2 the eight max-growth conditions have twice the weight of - the 33 flux conditions, which is the intended emphasis here: - the max-growth set is where the carbon-source diversity lives, - while 30 of the 33 flux conditions are glucose. - + At 2 the max-growth dataset counts twice as much as the flux + dataset in the combination. Returns ------- rmse : float Weighted mean of whichever dataset RMSEs are present. ``NaN`` - if neither dataset is present. + if neither dataset is present or no condition of the present + datasets has a weight other than 0. detail : dict[str, numpy.ndarray] Per-condition RMSE arrays, keyed ``"flux_data"`` / ``"max_growth"`` for whichever datasets were scored. @@ -225,8 +239,9 @@ def experiment_distance( constrain=True, carbon_weights=carbon_weights, bio_rxn_id=bio_rxn_id, penalty=penalty, ) - parts.append(rmse) - weights.append(1.0) + if not np.isnan(rmse): + parts.append(rmse) + weights.append(1.0) detail["flux_data"] = per_cond if experiment_data.max_growth is not None: @@ -237,8 +252,9 @@ def experiment_distance( constrain=False, carbon_weights=carbon_weights, bio_rxn_id=bio_rxn_id, penalty=penalty, ) - parts.append(rmse) - weights.append(float(max_growth_weight)) + if not np.isnan(rmse): + parts.append(rmse) + weights.append(float(max_growth_weight)) detail["max_growth"] = per_cond if parts: diff --git a/src/raven_toolbox/experiments/flux_data.py b/src/raven_toolbox/experiments/flux_data.py index 8231f25..8cd6685 100644 --- a/src/raven_toolbox/experiments/flux_data.py +++ b/src/raven_toolbox/experiments/flux_data.py @@ -40,9 +40,10 @@ class FluxData: Exchange reaction IDs in the model, matching the columns of ``exch_fluxes`` and ``exch_mets``. weight - Optional per-condition weights of the - distance to the measurements (0 leaves the condition out). ``None`` if the source file did not have the - ``weight`` column. + Optional per-condition weights of the distance to the + measurements. A weight of 0 leaves the condition out of the + simulation and of the distance. ``None`` if the source file did not + have the ``weight`` column. source Optional per-condition free-text describing where the data came from. ``None`` if the source file did not have a @@ -76,7 +77,8 @@ def load_flux_data(path: str | Path) -> FluxData: Optional extra columns (in any position): - * ``weight`` (float): weight of the condition in the distance to the measurements. + * ``weight`` (float): weight of the condition in the distance to the measurements; 0 + leaves the condition out of the simulation and of the distance. * ``source`` (str): free-text describing the data origin. An exchange column header has the form `` ()``, split at diff --git a/tests/test_experiments_conditions.py b/tests/test_experiments_conditions.py index 89e0397..78210d5 100644 --- a/tests/test_experiments_conditions.py +++ b/tests/test_experiments_conditions.py @@ -474,3 +474,141 @@ def optimize(m): assert len(calls) == 2 assert all(s.feasible for s in sims) + + +# --------------------------------------------------------------------------- # +# Conditions with weight 0 are left out +# --------------------------------------------------------------------------- # + +def _with_std_row(flux_data: FluxData) -> FluxData: + """The data with a first condition named ``Std`` that matches no exchange and has + weight 0, as the condition of a protein measurement in a tutorial file.""" + n_rxns = len(flux_data.exch_rxn_ids) + weight = np.ones(len(flux_data.conds)) if flux_data.weight is None else flux_data.weight + return FluxData( + conds=["Std"] + list(flux_data.conds), + p_tot=np.concatenate([[0.5], flux_data.p_tot]), + gr_rate=np.concatenate([[0.1], flux_data.gr_rate]), + exch_fluxes=np.vstack([np.full((1, n_rxns), np.nan), flux_data.exch_fluxes]), + exch_mets=list(flux_data.exch_mets), + exch_rxn_ids=list(flux_data.exch_rxn_ids), + weight=np.concatenate([[0.0], weight]), + ) + + +def _simulate(model, flux_data, *, constrain): + return simulate_conditions( + model, flux_data, + constrain=constrain, zero_flux_rxns=["EX_byp"] if constrain else [], + bio_rxn_id="biomass", + ) + + +def test_a_condition_with_weight_zero_is_not_simulated_or_matched(): + model = _build_toy() + flux_data = _flux_data(ethanol_grrate=20.0) + with_std = _with_std_row(flux_data) + + sims = _simulate(model, with_std, constrain=True) + reference = _simulate(_build_toy(), flux_data, constrain=True) + + assert len(sims) == 3 + assert sims[0].skipped and not sims[0].feasible + assert [s.skipped for s in sims[1:]] == [False, False] + assert [s.growth for s in sims[1:]] == [s.growth for s in reference] + + +def test_a_condition_with_weight_zero_does_not_block_its_exchange(): + model = _build_toy() + flux_data = _flux_data(ethanol_grrate=20.0) + seen = [] + + def optimize(m): + seen.append(m.reactions.get_by_id("EX_eth").lower_bound) + return m.optimize() + + flux_data.weight = np.array([1.0, 0.0]) + _simulate_with(model, flux_data, optimize) + assert seen == [-1000.0] + + seen.clear() + flux_data.weight = np.array([1.0, 1.0]) + _simulate_with(model, flux_data, optimize) + assert seen == [0.0, -20.0] + + +def _simulate_with(model, flux_data, optimize): + return simulate_conditions( + model, flux_data, constrain=True, zero_flux_rxns=[], bio_rxn_id="biomass", + optimize=optimize, + ) + + +@pytest.mark.parametrize("constrain", [True, False]) +def test_a_condition_with_weight_zero_does_not_change_the_distance(constrain): + model = _build_toy() + flux_data = _flux_data(ethanol_grrate=20.0) if constrain else _max_growth_data() + with_std = _with_std_row(flux_data) + carbon_weights = compute_carbon_weights( + model, ["EX_glc", "EX_eth", "EX_byp", "biomass"], bio_rxn_id="biomass", + ) + + rmse, per_cond = dataset_rmse( + with_std, _simulate(model, with_std, constrain=constrain), + constrain=constrain, carbon_weights=carbon_weights, bio_rxn_id="biomass", + ) + expected, expected_per_cond = dataset_rmse( + flux_data, _simulate(_build_toy(), flux_data, constrain=constrain), + constrain=constrain, carbon_weights=carbon_weights, bio_rxn_id="biomass", + ) + + assert rmse == expected + assert np.isnan(per_cond[0]) + assert list(per_cond[1:]) == list(expected_per_cond) + + +def test_experiment_distance_leaves_out_a_dataset_without_active_conditions(): + model = _build_toy() + flux_data = _flux_data(ethanol_grrate=20.0) + flux_data.weight = np.array([0.0, 0.0]) + max_growth = _max_growth_data() + experiment_data = ExperimentData(flux_data=flux_data, max_growth=max_growth) + carbon_weights = compute_carbon_weights( + model, ["EX_glc", "EX_eth", "biomass"], bio_rxn_id="biomass", + ) + flux_sims = _simulate(model, flux_data, constrain=True) + max_growth_sims = _simulate(model, max_growth, constrain=False) + assert all(s.skipped for s in flux_sims) + + rmse, detail = experiment_distance( + experiment_data, flux_sims=flux_sims, max_growth_sims=max_growth_sims, + carbon_weights=carbon_weights, bio_rxn_id="biomass", + ) + expected, _ = dataset_rmse( + max_growth, max_growth_sims, constrain=False, carbon_weights=carbon_weights, + bio_rxn_id="biomass", + ) + + assert rmse == expected + assert np.isnan(detail["flux_data"]).all() + + +def test_the_distance_is_nan_when_every_condition_has_weight_zero(): + model = _build_toy() + flux_data = _flux_data(ethanol_grrate=20.0) + flux_data.weight = np.array([0.0, 0.0]) + carbon_weights = compute_carbon_weights( + model, ["EX_glc", "EX_eth", "biomass"], bio_rxn_id="biomass", + ) + sims = _simulate(model, flux_data, constrain=True) + + rmse, per_cond = dataset_rmse( + flux_data, sims, constrain=True, carbon_weights=carbon_weights, bio_rxn_id="biomass", + ) + distance, _ = experiment_distance( + ExperimentData(flux_data=flux_data), flux_sims=sims, max_growth_sims=None, + carbon_weights=carbon_weights, bio_rxn_id="biomass", + ) + + assert np.isnan(rmse) and np.isnan(per_cond).all() + assert np.isnan(distance) From 6c460d5390141b578f86b2a8e21f5a7fc6f0b595 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Fri, 20 Nov 2026 23:17:01 +0100 Subject: [PATCH 6/6] test: cover the errors and the NGAM bounds of set_gam with keep_offsets_of The same cases as the tests of setGAM with keepOffsetsOf in RAVEN. --- tests/test_biomass_maintenance.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_biomass_maintenance.py b/tests/test_biomass_maintenance.py index 9f6ba19..0266c77 100644 --- a/tests/test_biomass_maintenance.py +++ b/tests/test_biomass_maintenance.py @@ -104,6 +104,24 @@ def test_keep_offsets_reverts_in_a_context(): assert model.reactions.NGAM.bounds == (1.0, 1.0) +def test_keep_offsets_of_needs_one_metabolite_of_that_name_in_the_reaction(): + with pytest.raises(ValueError, match="needs exactly one"): + set_gam(_toy(), 45.0, biomass_rxn="BIO", cofactor_met_names=("nothing", "ATP"), + keep_offsets_of="nothing") + + +def test_get_gam_needs_the_biomass_reaction(): + with pytest.raises(KeyError): + get_gam(_toy(), biomass_rxn="NOPE") + + +def test_keep_offsets_sets_ngam_too(): + model = _toy(gam=30.0) + set_gam(model, 45.0, biomass_rxn="BIO", cofactor_met_names=NAMES, + ngam_rxn="NGAM", ngam_value=2.0, keep_offsets_of="ATP") + assert model.reactions.NGAM.bounds == (2.0, 2.0) + + def test_keep_offsets_of_must_be_a_cofactor(): with pytest.raises(ValueError, match="not one of the cofactor names"): set_gam(_toy(), 45.0, biomass_rxn="BIO", cofactor_met_names=("ADP",), keep_offsets_of="ATP")