diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d3ac5c..5b033ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,24 @@ * `get_model_from_homology` with `complex_policy="drop"` removes each complex that has a subunit without an ortholog, and removes the reaction only when no complex remains (was: the reaction was removed as soon as one subunit had no ortholog). For the rule `(A and B) or (C and D)` without an ortholog of `B`, the draft has `C and D`. * `manipulation.contract_model` is the new name of `remove_duplicate_reactions`, after RAVEN's `contractModel`, and the old name is removed. `contract_model` takes `dist_reverse` (default `True`), the `distReverse` option of `contractModel`: with `False`, reactions with the same metabolites and a different reversibility are duplicates. +### Experiments + +* New: the `experiments` package scores a model against measured growth rates and exchange fluxes. `load_experiment_data` reads `fluxData.tsv`, `maxGrowth.tsv` and `zeroFlux.tsv` from a folder, `load_flux_data` parses one flux file into a `FluxData` (with a per-condition `weight`), `simulate_conditions` fixes the uptake of each condition and solves the model, `compute_carbon_weights` gives each exchange the carbon count of its metabolite, `experiment_distance` combines the carbon-weighted RMSE of flux and max-growth conditions into one value, and `apply_flux_data_constraints` sets the exchange bounds of a model from one condition (loose, or a percentage band). The organism-specific steps, an anaerobic switch and a change of the protein content, and the solver call are passed as functions. A condition with weight 0 is left out of the simulation and the distance. + +### Solver + +* New: `utils.solver` holds the support for optlang's scipy (HiGHS) interface, next to `set_raven_tolerance`. `repair_scipy_problem` renumbers the variable columns after a removal and rebuilds a stale coefficient matrix. `switch_to_scipy` and `edit_on_fast_solver` avoid the dense-matrix cost of building or editing a large model. `optimize_model` returns a solution with its status for an infeasible or unbounded LP instead of raising, and `constraint_shadow_prices` reads the row marginals on the scipy interface. + +### Biomass + +* New: `biomass.fit_maintenance` fits NGAM from a maintenance coefficient and GAM from chemostat substrate uptake, with a study-weighted Huber loss and a bootstrap over studies. The steps are also available separately (`fit_gam`, `bootstrap_gam`, `uptake_grid`, `min_uptake`, `ngam_from_maintenance_coefficient`, `study_weights`, `huber_loss`, `log_ratio`), and `MaintenanceConfig` holds the reaction ids and metabolite names. +* New: `get_gam`, and `set_gam` takes `keep_offsets_of`: the coefficient of that metabolite becomes the value and the other GAM terms shift by the same amount, so a charge-balancing proton offset stays. + +### Proteins and annotation + +* New: `utils.protein.calculate_mw` computes the molecular weight of a protein from its amino acid sequence (average residue masses plus one water; the codes B, J, O, U, X and Z are recognised; an empty sequence gives NaN). +* New: `annotation.find_met_smiles` fills `annotation["smiles"]` of the metabolites from PubChem, with a local TSV cache (`cache_path`) that is appended to after each lookup, and `annotation.met_smiles` reads the SMILES of a metabolite whether the annotation is a string or a list. + ### Data * The KEGG assets are the `kegg120` release of `toolbox-assets` (was: `kegg118`): the reference model and tables, the taxonomy and the profile HMM libraries of 4246 prokaryotic and 2724 eukaryotic KOs. The baked registry and `data/manifest.json` list them. diff --git a/src/raven_toolbox/annotation/__init__.py b/src/raven_toolbox/annotation/__init__.py index 7b0cc5b..b0a742a 100644 --- a/src/raven_toolbox/annotation/__init__.py +++ b/src/raven_toolbox/annotation/__init__.py @@ -1,4 +1,4 @@ -"""Annotation helpers: SBO term assignment and ΔG CSV persistence. +"""Annotation helpers: SBO term assignment, ΔG CSV persistence and SMILES lookup. These are the parts of yeast-GEM's ``missingFields`` module that are organism-agnostic. Default parameter values @@ -7,6 +7,7 @@ overrides. """ from raven_toolbox.annotation.delta_g import load_delta_g_csv, save_delta_g_csv +from raven_toolbox.annotation.smiles import find_met_smiles, met_smiles from raven_toolbox.annotation.sbo import ( DEFAULT_BIOMASS_MET_NAMES, DEFAULT_BIOMASS_RXN_NAME, @@ -19,6 +20,8 @@ "DEFAULT_BIOMASS_RXN_NAME", "DEFAULT_NGAM_RXN_NAME", "add_sbo_terms", + "find_met_smiles", "load_delta_g_csv", + "met_smiles", "save_delta_g_csv", ] diff --git a/src/raven_toolbox/annotation/smiles.py b/src/raven_toolbox/annotation/smiles.py new file mode 100644 index 0000000..7aa5bb5 --- /dev/null +++ b/src/raven_toolbox/annotation/smiles.py @@ -0,0 +1,233 @@ +"""SMILES lookup via PubChem with a local TSV cache. + +The cache file is a two-column TSV with no header: ``namesmiles``. +Each successful lookup is appended immediately so interrupted runs +resume gracefully. +""" +from __future__ import annotations + +import logging +import re +import time +from pathlib import Path +from typing import TYPE_CHECKING +from urllib.parse import quote + +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +if TYPE_CHECKING: + import cobra + + +__all__ = ["find_met_smiles", "met_smiles"] + +logger = logging.getLogger(__name__) + +# PubChem deprecated the `CanonicalSMILES` property in 2025 in favour of +# `SMILES` (its canonical/absolute SMILES); `ConnectivitySMILES` replaces +# the old `IsomericSMILES`. +_PUBCHEM_URL = ( + "https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/name/" + "{name}/property/SMILES/TXT" +) +_TIMEOUT_SECONDS = 30 +_NUM_RETRIES = 10 +_INTER_REQUEST_SLEEP = 0.05 # seconds; PubChem allows ~5 req/s +_PROT_PATTERN = re.compile(r"^prot_") +_FIRST_LINE_PATTERN = re.compile(r"^(\S*)\n") + + +def met_smiles(metabolite) -> str: + """The metabolite's SMILES, whatever shape its annotation is in. + + cobra stores an annotation value as the object it was assigned, and + returns a *list* for a model read back from a file: a metabolite + annotated in memory by :func:`find_met_smiles` has ``"CCO"``, + and the same metabolite after it is saved and loaded has + ``["CCO"]``. Readers that assume a string get a list that compares + equal to nothing and is not hashable. + + Returns the first non-empty entry, or ``""`` when there is none. + """ + value = metabolite.annotation.get("smiles", "") + if isinstance(value, (list, tuple, set)): + return next((str(v) for v in value if v), "") + return str(value) if value else "" + + +def find_met_smiles( + model: "cobra.Model", + *, + cache_path: str | Path, +) -> None: + """Populate ``metabolite.annotation['smiles']`` from PubChem. + + For every metabolite in the model: + + 1. Skip if the name starts with ``prot_`` (enzyme pseudometabolite). + 2. Skip, with a warning, if the metabolite has no name. + 3. Skip if ``annotation['smiles']`` is already set. + 4. Look up the metabolite name in the local TSV cache. When the cache + holds a name more than once, the last row is used. + 5. If still not found, query PubChem; cache the result. + + The same metabolite name may appear on many ``cobra.Metabolite`` + instances (different compartments). Lookup is done once per unique + name and the SMILES is applied to every metabolite sharing that name. + + Parameters + ---------- + model + A cobra.Model. Mutated in place: each metabolite gains + ``annotation['smiles']`` when a SMILES is found. + cache_path + Path to the SMILES cache TSV. The file is created if missing and + appended to as new SMILES are downloaded. + """ + cache_path = Path(cache_path) + cache_path.parent.mkdir(parents=True, exist_ok=True) + + # Build map: metabolite name -> list of metabolite objects sharing it. + name_to_mets: dict[str, list] = {} + unnamed = [] + for met in model.metabolites: + name = met.name + if not name: + unnamed.append(met.id) + continue + if _PROT_PATTERN.match(name): + continue + if met.annotation.get("smiles"): + continue + name_to_mets.setdefault(name, []).append(met) + + if unnamed: + logger.warning( + "find_met_smiles: %d metabolite(s) have no name and get no SMILES: %s", + len(unnamed), ", ".join(unnamed), + ) + + if not name_to_mets: + logger.info("find_met_smiles: nothing to look up.") + return + + # Load existing cache. + cache = _load_cache(cache_path) + logger.info( + "find_met_smiles: %d unique metabolite name(s) need SMILES; " + "%d already in cache.", + len(name_to_mets), + sum(1 for n in name_to_mets if n in cache), + ) + + # Apply cached SMILES first (no network). + for name in list(name_to_mets): + if name in cache: + smiles = cache[name] + for met in name_to_mets[name]: + if smiles: + met.annotation["smiles"] = smiles + del name_to_mets[name] + + if not name_to_mets: + logger.info("find_met_smiles: all matches resolved from cache.") + return + + # Hit PubChem for the rest. + session = _make_session() + n_found = 0 + n_missing = 0 + + for name in sorted(name_to_mets): + smiles = _fetch_one_smiles(session, name) + # Append to cache immediately, including empty results so we + # do not retry next time. + _append_cache(cache_path, name, smiles) + + if smiles: + for met in name_to_mets[name]: + met.annotation["smiles"] = smiles + n_found += 1 + else: + n_missing += 1 + logger.debug("No SMILES for: %s", name) + + time.sleep(_INTER_REQUEST_SLEEP) + + total = n_found + n_missing + if total > 0: + pct = 100.0 * n_found / total + logger.info( + "find_met_smiles: SMILES found for %d/%d unique names (%.0f%%); " + "%d missing.", + n_found, total, pct, n_missing, + ) + + +def _load_cache(path: Path) -> dict[str, str]: + """Load the two-column TSV cache. Returns an empty dict if missing.""" + cache: dict[str, str] = {} + if not path.is_file(): + return cache + with open(path, "r", encoding="utf-8") as f: + for line in f: + parts = line.rstrip("\n").split("\t") + if len(parts) >= 2: + cache[parts[0]] = parts[1] + elif len(parts) == 1 and parts[0]: + cache[parts[0]] = "" + return cache + + +def _append_cache(path: Path, name: str, smiles: str) -> None: + """Append one row to the TSV cache. Open and close per call so + interrupted runs leave a consistent file.""" + with open(path, "a", encoding="utf-8") as f: + f.write(f"{name}\t{smiles}\n") + + +def _make_session() -> requests.Session: + """A requests.Session with retry/backoff for transient errors.""" + session = requests.Session() + retry = Retry( + total=_NUM_RETRIES, + backoff_factor=0.5, + status_forcelist=(429, 500, 502, 503, 504), + allowed_methods=("GET",), + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount("https://", adapter) + session.mount("http://", adapter) + return session + + +def _fetch_one_smiles(session: requests.Session, name: str) -> str: + """Fetch SMILES from PubChem for one metabolite name. Returns + empty string if unfound or unrecoverable error.""" + url = _PUBCHEM_URL.format(name=quote(name, safe="")) + try: + response = session.get(url, timeout=_TIMEOUT_SECONDS) + except requests.RequestException as e: + logger.warning("PubChem request failed for %r: %s", name, e) + return "" + + if response.status_code in (400, 404, 500): + return "" + if response.status_code != 200: + logger.warning( + "PubChem returned HTTP %d for %r; treating as no match.", + response.status_code, name, + ) + return "" + + text = response.text + if not text: + return "" + + # Multiple SMILES may come on separate lines; take only the first. + match = _FIRST_LINE_PATTERN.match(text) + if match: + return match.group(1).strip() + return text.split("\n", 1)[0].strip() diff --git a/src/raven_toolbox/biomass/__init__.py b/src/raven_toolbox/biomass/__init__.py index c0fc62a..12f12c3 100644 --- a/src/raven_toolbox/biomass/__init__.py +++ b/src/raven_toolbox/biomass/__init__.py @@ -26,6 +26,9 @@ scale_biomass(model, cfg, "protein", 0.50, balance_out="carbohydrate") set_gam(model, 80, biomass_rxn=cfg.biomass_rxn, cofactor_met_names=("ATP", "ADP", "H2O", "H+", "phosphate")) + +GAM and NGAM are fitted to chemostat uptakes and a maintenance coefficient with +:func:`fit_maintenance` (see :mod:`raven_toolbox.biomass.maintenance`). """ from raven_toolbox.biomass.config import BiomassComponent, BiomassConfig from raven_toolbox.biomass.fit_parameters import ( @@ -33,7 +36,20 @@ ParameterPosition, fit_parameters, ) -from raven_toolbox.biomass.gam import set_gam +from raven_toolbox.biomass.gam import get_gam, set_gam +from raven_toolbox.biomass.maintenance import ( + MaintenanceConfig, + MaintenanceFit, + bootstrap_gam, + fit_gam, + fit_maintenance, + huber_loss, + log_ratio, + min_uptake, + ngam_from_maintenance_coefficient, + study_weights, + uptake_grid, +) from raven_toolbox.biomass.scale import ( rescale_pseudoreaction, scale_biomass, @@ -44,10 +60,22 @@ "BiomassComponent", "BiomassConfig", "FitParametersResult", + "MaintenanceConfig", + "MaintenanceFit", "ParameterPosition", + "bootstrap_gam", + "fit_gam", + "fit_maintenance", "fit_parameters", + "get_gam", + "huber_loss", + "log_ratio", + "min_uptake", + "ngam_from_maintenance_coefficient", "rescale_pseudoreaction", "scale_biomass", "set_gam", + "study_weights", "sum_biomass", + "uptake_grid", ] diff --git a/src/raven_toolbox/biomass/gam.py b/src/raven_toolbox/biomass/gam.py index d054576..9f5d8cf 100644 --- a/src/raven_toolbox/biomass/gam.py +++ b/src/raven_toolbox/biomass/gam.py @@ -1,6 +1,9 @@ """Growth-associated maintenance (GAM) and non-growth maintenance (NGAM). -Ports the generic part of yeast-GEM's ``changeGAM.m``. +Ports the generic part of yeast-GEM's ``changeGAM.m``. GAM is the ATP hydrolysed per gram +of biomass formed (mmol/gDW), held in the coefficients of the ATP hydrolysis terms of the +biomass pseudoreaction. NGAM is the ATP hydrolysed per hour and gram of biomass +(mmol/gDW/h), held as the fixed flux of the maintenance reaction. """ from __future__ import annotations @@ -17,6 +20,7 @@ def set_gam( cofactor_met_names: Iterable[str], ngam_rxn: str | None = None, ngam_value: float | None = None, + keep_offsets_of: str | None = None, ) -> cobra.Model: """Set GAM (and optionally NGAM) on a model in place. @@ -25,6 +29,12 @@ def set_gam( H+, phosphate) to ``±value``, keeping the sign of its current coefficient. + With ``keep_offsets_of`` the coefficients keep their differences instead: + the coefficient of the metabolite of that name becomes ``±value`` and every + other cofactor changes by the same amount in the direction of its sign. A + proton coefficient that balances the charge of the other biomass terms then + keeps its offset from the ATP coefficient. + If both ``ngam_rxn`` and ``ngam_value`` are given, the NGAM reaction's bounds are fixed at ``(ngam_value, ngam_value)``. An empty ``ngam_rxn`` counts as not given, and ``ngam_value`` without an ``ngam_rxn`` is an @@ -43,6 +53,11 @@ def set_gam( Iterable of metabolite names (not ids). Every metabolite in the biomass pseudoreaction whose name is in this set is rescaled. yeast-GEM uses ``{"ATP", "ADP", "H2O", "H+", "phosphate"}``. + keep_offsets_of + Name of the metabolite whose coefficient becomes ``±value`` while the + other cofactors shift by the same amount (see above). It must be one of + ``cofactor_met_names`` and in the biomass pseudoreaction. ``None`` + sets every cofactor to ``±value``. ngam_rxn, ngam_value Optional NGAM update. If both are provided, the reaction's bounds are set to ``(ngam_value, ngam_value)`` (equality @@ -59,7 +74,8 @@ def set_gam( If ``biomass_rxn``, or a given ``ngam_rxn``, is not in the model. Both reactions are looked up before anything is changed. ValueError - If ``ngam_value`` is given without an ``ngam_rxn``. + If ``ngam_value`` is given without an ``ngam_rxn``, or ``keep_offsets_of`` + names no cofactor of the biomass pseudoreaction. """ if ngam_value is not None and not ngam_rxn: raise ValueError("ngam_value needs an ngam_rxn: the NGAM reaction to set is not given") @@ -68,14 +84,44 @@ def set_gam( cofactor_set = set(cofactor_met_names) targets: dict[cobra.Metabolite, float] = {} + shift = 0.0 + if keep_offsets_of is not None: + shift = float(value) - _gam_coefficient(rxn, keep_offsets_of, cofactor_set) for met, coef in rxn.metabolites.items(): if met.name in cofactor_set: - target = (1 if coef > 0 else -1) * float(value) - targets[met] = target - if targets: + sign = 1 if coef > 0 else -1 + targets[met] = sign * shift if keep_offsets_of is not None else sign * float(value) + if targets and keep_offsets_of is not None: + rxn.add_metabolites(targets, combine=True) # adds the shift, which a context reverts + elif targets: rxn.add_metabolites(targets, combine=False) # replaces the coefficient, so it is exactly target if ngam is not None and ngam_value is not None: ngam.bounds = (float(ngam_value), float(ngam_value)) return model + + +def get_gam(model: cobra.Model, *, biomass_rxn: str, atp_met_name: str = "ATP") -> float: + """GAM of the model: the size of the ATP coefficient of the biomass pseudoreaction. + + Raises + ------ + KeyError + If ``biomass_rxn`` is not in the model. + ValueError + If the pseudoreaction has no metabolite named ``atp_met_name``. + """ + rxn = model.reactions.get_by_id(biomass_rxn) + return _gam_coefficient(rxn, atp_met_name, {atp_met_name}) + + +def _gam_coefficient(rxn: cobra.Reaction, met_name: str, cofactor_names: set[str]) -> float: + if met_name not in cofactor_names: + raise ValueError(f"keep_offsets_of {met_name!r} is not one of the cofactor names") + found = [abs(coef) for met, coef in rxn.metabolites.items() if met.name == met_name] + if len(found) != 1: + raise ValueError( + f"{rxn.id} has {len(found)} metabolites named {met_name!r}; the GAM term needs exactly one" + ) + return found[0] diff --git a/src/raven_toolbox/biomass/maintenance.py b/src/raven_toolbox/biomass/maintenance.py new file mode 100644 index 0000000..8bcb4e3 --- /dev/null +++ b/src/raven_toolbox/biomass/maintenance.py @@ -0,0 +1,357 @@ +"""Fit GAM and NGAM to measured substrate uptake and maintenance coefficients. + +GAM (the ATP hydrolysed per gram of biomass formed) and NGAM (the ATP hydrolysed per hour +and gram of biomass) are fixed by different data, so they are fitted one after the other: + +* NGAM from the maintenance coefficient measured in retentostats, or in chemostats at very + low growth rates. The substrate uptake that a model needs at zero growth is proportional + to NGAM, so :func:`ngam_from_maintenance_coefficient` needs one solve. +* GAM from the substrate uptake of chemostats at the measured growth rates, with NGAM held + at its value (:func:`fit_gam`). The model uptake is computed once on a grid of GAM values + (:func:`uptake_grid`); the loss, the best GAM and the resampling of studies + (:func:`bootstrap_gam`) then work on that grid and need no further solves. + +The chemostat data come from many studies. The loss weights every study equally +(:func:`study_weights`: a unit counts ``1 / sqrt`` of the number of units of its study, and +units from a less reliable origin count less) and is the Huber loss of the log ratio of model +to measured uptake (:func:`huber_loss`, :func:`log_ratio`), so that one study that +disagrees with the others does not move the result. + +:func:`fit_maintenance` runs the steps for one data set. A model that has several regimes, +for example aerobic and anaerobic growth, is fitted once per regime with the model as it is +used in that regime. + +The ids of the reactions and the names of the metabolites are given in a +:class:`MaintenanceConfig`. A step that depends on the data, such as setting the protein +content of the biomass of a chemostat, is a function in ``condition_setup``. +""" +from __future__ import annotations + +from collections.abc import Callable, Iterable, Sequence +from dataclasses import dataclass, field + +import cobra +import numpy as np +import pandas as pd + +from raven_toolbox.biomass.gam import get_gam, set_gam + +__all__ = [ + "DEFAULT_GAMS", + "HUBER_DELTA", + "INFEASIBLE_LOG_RATIO", + "MaintenanceConfig", + "MaintenanceFit", + "bootstrap_gam", + "fit_gam", + "fit_maintenance", + "huber_loss", + "log_ratio", + "min_uptake", + "ngam_from_maintenance_coefficient", + "study_weights", + "uptake_grid", +] + +#: Transition of the Huber loss, as a log ratio: residuals below 0.15 (about 15%) count +#: quadratically, larger ones linearly. +HUBER_DELTA = 0.15 +#: Log ratio assigned to a growth rate that the model cannot sustain (about a factor 2). +INFEASIBLE_LOG_RATIO = 0.7 +#: GAM values (mmol/gDW) at which the model uptake is computed. +DEFAULT_GAMS = np.arange(25.0, 70.01, 1.25) + + +@dataclass(frozen=True) +class MaintenanceConfig: + """Reactions and metabolites that carry GAM and NGAM in a model. + + Attributes + ---------- + biomass_rxn: + Biomass pseudoreaction that holds the GAM coefficients. + ngam_rxn: + Maintenance reaction whose fixed flux is NGAM. + growth_rxn: + Reaction whose flux is the growth rate, fixed to the dilution rate of a chemostat. + Often the same as ``biomass_rxn``, or a reaction that drains the biomass. + substrate_exchange: + Exchange reaction of the limiting substrate. Its uptake is a negative flux. + cofactor_met_names: + Names of the metabolites of the GAM term in the biomass pseudoreaction, such as ATP, + ADP, H2O, H+ and phosphate. + atp_met_name: + Name of the ATP metabolite, whose coefficient is the GAM. + """ + + biomass_rxn: str + ngam_rxn: str + growth_rxn: str + substrate_exchange: str + cofactor_met_names: tuple[str, ...] + atp_met_name: str = "ATP" + + +@dataclass +class MaintenanceFit: + """The values fitted to one data set. + + Attributes + ---------- + ngam, gam: + Fitted NGAM and GAM. + gam_p16, gam_p84: + 16th and 84th percentile of the best GAM when the studies are resampled. + units: + Number of data rows. + studies: + Number of studies. + median_error: + Median relative difference between model and measured uptake at the fitted GAM. + """ + + ngam: float + gam: float + gam_p16: float + gam_p84: float + units: int + studies: int + median_error: float + + +def huber_loss(residual: np.ndarray, delta: float = HUBER_DELTA) -> np.ndarray: + """Huber loss: ``r**2 / 2`` for ``|r| <= delta``, ``delta * (|r| - delta / 2)`` above.""" + r = np.asarray(residual, dtype=float) + a = np.abs(r) + return np.where(a <= delta, 0.5 * r * r, delta * (a - 0.5 * delta)) + + +def log_ratio( + simulated: np.ndarray, measured: np.ndarray, infeasible: float = INFEASIBLE_LOG_RATIO, +) -> np.ndarray: + """Natural log of ``simulated / measured``, row-wise broadcast over ``simulated``. + + A value that is not finite, such as the NaN of a growth rate the model cannot sustain, + gives ``infeasible``. + """ + with np.errstate(all="ignore"): + r = np.log(np.asarray(simulated, dtype=float) / np.asarray(measured, dtype=float)[None, :]) + return np.where(np.isfinite(r), r, infeasible) + + +def study_weights( + study: Iterable, *, discounted: Iterable[bool] | None = None, discount: float = 0.5, +) -> np.ndarray: + """Weight of every data row: ``1 / sqrt(rows of its study)``, times ``discount`` where + ``discounted`` is True (for example values read from a figure).""" + counts = pd.Series(list(study)).groupby(list(study)).transform("size").to_numpy(dtype=float) + weight = 1.0 / np.sqrt(counts) + if discounted is not None: + weight = weight * np.where(np.asarray(list(discounted), dtype=bool), discount, 1.0) + return weight + + +def min_uptake( + model: cobra.Model, + growth: float, + config: MaintenanceConfig, + *, + setup: Callable[[cobra.Model], None] | None = None, +) -> float: + """Smallest substrate uptake (a positive number) at a fixed growth rate. + + The model is changed only inside ``with model:``, so it is as before after the call. + ``setup`` is called on the model before the solve. The result is NaN when the growth + rate cannot be reached. + """ + with model: + if setup is not None: + setup(model) + substrate = model.reactions.get_by_id(config.substrate_exchange) + substrate.lower_bound = -1000.0 + model.reactions.get_by_id(config.growth_rxn).bounds = (growth, growth) + model.objective = {substrate: 1.0} # an uptake is a negative flux + value = model.slim_optimize(error_value=np.nan) + return -value + + +def uptake_grid( + model: cobra.Model, + data: pd.DataFrame, + gams: Sequence[float], + ngam: float, + config: MaintenanceConfig, + *, + growth_col: str = "growth_rate", + condition_setup: Callable[[cobra.Model, pd.Series], None] | None = None, +) -> np.ndarray: + """Model uptake for every GAM in ``gams`` (rows) and data row (columns). + + ``condition_setup(model, row)`` is called on the model for each row, for example to set + the protein content that was measured in that chemostat. + """ + out = np.full((len(gams), len(data)), np.nan) + growth = data[growth_col].to_numpy(dtype=float) + rows = [data.iloc[j] for j in range(len(data))] + for i, gam in enumerate(gams): + with model: + _set_maintenance(model, float(gam), ngam, config) + for j, row in enumerate(rows): + setup = None if condition_setup is None else (lambda m, r=row: condition_setup(m, r)) + out[i, j] = min_uptake(model, growth[j], config, setup=setup) + return out + + +def ngam_from_maintenance_coefficient( + model: cobra.Model, + ms_substrate: float, + config: MaintenanceConfig, + *, + gam: float | None = None, +) -> float: + """NGAM at which the model needs ``ms_substrate`` (mmol/gDW/h) at zero growth. + + The uptake at zero growth is proportional to NGAM, so one solve is enough. The ATP yield + of the substrate comes from the model. ``gam`` is set first when given. + """ + with model: + _set_maintenance(model, gam, 1.0, config) + per_unit = min_uptake(model, 0.0, config) + return float(ms_substrate / per_unit) + + +def fit_gam( + model: cobra.Model, + data: pd.DataFrame, + ngam: float, + config: MaintenanceConfig, + *, + gams: Sequence[float] | None = None, + growth_col: str = "growth_rate", + uptake_col: str = "uptake", + weights: np.ndarray | None = None, + condition_setup: Callable[[cobra.Model, pd.Series], None] | None = None, +) -> tuple[float, np.ndarray, np.ndarray]: + """Best GAM for the rows of ``data`` at a given NGAM. + + ``weights`` weight the rows (default: all 1; see :func:`study_weights`). Returns the GAM, + the GAM grid, and the matrix of log ratios (GAM by row of ``data``), which + :func:`bootstrap_gam` resamples without new solves. + """ + gams = np.asarray(DEFAULT_GAMS if gams is None else gams, dtype=float) + measured = data[uptake_col].to_numpy(dtype=float) + ratios = log_ratio( + uptake_grid(model, data, gams, ngam, config, growth_col=growth_col, + condition_setup=condition_setup), + measured, + ) + w = np.ones(len(data)) if weights is None else np.asarray(weights, dtype=float) + return _best_gam(gams, ratios, w), gams, ratios + + +def bootstrap_gam( + gams: Sequence[float], + ratios: np.ndarray, + study: Iterable, + weights: np.ndarray, + *, + n: int = 400, + seed: int = 0, + percentiles: tuple[float, float] = (16.0, 84.0), +) -> tuple[float, float]: + """Percentiles of the best GAM when the studies are resampled with replacement. + + ``study`` gives the study of every column of ``ratios``; a resample draws as many studies + as there are, and a row counts as often as its study was drawn. + """ + rng = np.random.default_rng(seed) + gams = np.asarray(gams, dtype=float) + labels = np.asarray(list(study)) + names = sorted(set(labels)) + best = [] + for _ in range(n): + drawn = rng.choice(len(names), len(names)) + count = np.zeros(len(labels)) + for k in drawn: + count[labels == names[k]] += 1 + w = weights * count + if w.sum() == 0: + continue + best.append(_best_gam(gams, ratios, w)) + lo, hi = np.percentile(best, percentiles) + return float(lo), float(hi) + + +def fit_maintenance( + model: cobra.Model, + data: pd.DataFrame, + ms_substrate: float, + config: MaintenanceConfig, + *, + study_col: str = "study", + discounted: Iterable[bool] | None = None, + discount: float = 0.5, + growth_col: str = "growth_rate", + uptake_col: str = "uptake", + gams: Sequence[float] | None = None, + bootstrap: int = 400, + seed: int = 0, + condition_setup: Callable[[cobra.Model, pd.Series], None] | None = None, +) -> MaintenanceFit: + """NGAM from the maintenance coefficient and GAM from the chemostats of ``data``. + + Parameters + ---------- + model: + Model as it is used for the growth regime of ``data``. It is not changed. + data: + One row per chemostat, with the growth rate (``growth_col``), the measured substrate + uptake (``uptake_col``) and the study (``study_col``). + ms_substrate: + Maintenance coefficient of the substrate, mmol/gDW/h, from retentostats. + config: + The reactions and metabolites of GAM and NGAM. + discounted, discount: + Rows that count ``discount`` times as much in the weights, for example values read + from figures. + gams: + GAM values of the grid (default :data:`DEFAULT_GAMS`). + bootstrap, seed: + Number of resamples of the studies and the seed. + condition_setup: + Function ``(model, row)`` that sets the model for the conditions of one chemostat. + """ + data = data.reset_index(drop=True) + ngam = ngam_from_maintenance_coefficient(model, ms_substrate, config) + study = data[study_col].to_numpy() + weights = study_weights(study, discounted=discounted, discount=discount) + gam, grid, ratios = fit_gam( + model, data, ngam, config, gams=gams, growth_col=growth_col, + uptake_col=uptake_col, weights=weights, condition_setup=condition_setup, + ) + lo, hi = bootstrap_gam(grid, ratios, study, weights, n=bootstrap, seed=seed) + best = ratios[int(np.argmin(np.abs(grid - gam)))] + return MaintenanceFit( + ngam=ngam, gam=gam, gam_p16=lo, gam_p84=hi, + units=len(data), studies=len(set(study)), + median_error=float(np.median(np.abs(np.exp(best) - 1.0))), + ) + + +def _best_gam(gams: np.ndarray, ratios: np.ndarray, weights: np.ndarray) -> float: + loss = (huber_loss(ratios) * weights[None, :]).sum(axis=1) / weights.sum() + return float(gams[int(np.argmin(loss))]) + + +def _set_maintenance( + model: cobra.Model, gam: float | None, ngam: float, config: MaintenanceConfig, +) -> None: + """Set GAM (when given) and NGAM in place, keeping the offsets between the GAM terms.""" + if gam is None: + gam = get_gam(model, biomass_rxn=config.biomass_rxn, atp_met_name=config.atp_met_name) + set_gam( + model, gam, biomass_rxn=config.biomass_rxn, + cofactor_met_names=config.cofactor_met_names, + ngam_rxn=config.ngam_rxn, ngam_value=ngam, + keep_offsets_of=config.atp_met_name, + ) diff --git a/src/raven_toolbox/experiments/__init__.py b/src/raven_toolbox/experiments/__init__.py new file mode 100644 index 0000000..272f498 --- /dev/null +++ b/src/raven_toolbox/experiments/__init__.py @@ -0,0 +1,35 @@ +"""Experimental data on growth and fluxes, and how a model is scored against it. + +The measurements of several conditions are read from files +(:func:`load_experiment_data`), one condition is set up in the model and +simulated (:func:`simulate_conditions`), and the distance between simulated and +measured values is computed (:func:`experiment_distance`). None of this needs +more than a :class:`cobra.Model`; setting that depends on the organism, such as +an anaerobic switch, is passed as a function. +""" +from .conditions import ConditionSimResult, simulate_conditions +from .constrain import apply_flux_data_constraints +from .data import ExperimentData, load_experiment_data +from .distance import ( + BIOMASS_CARBON_EQUIV, + INFEASIBLE_PENALTY, + compute_carbon_weights, + dataset_rmse, + experiment_distance, +) +from .flux_data import FluxData, load_flux_data + +__all__ = [ + "BIOMASS_CARBON_EQUIV", + "INFEASIBLE_PENALTY", + "ConditionSimResult", + "ExperimentData", + "FluxData", + "apply_flux_data_constraints", + "compute_carbon_weights", + "dataset_rmse", + "experiment_distance", + "load_experiment_data", + "load_flux_data", + "simulate_conditions", +] diff --git a/src/raven_toolbox/experiments/conditions.py b/src/raven_toolbox/experiments/conditions.py new file mode 100644 index 0000000..dfb5cab --- /dev/null +++ b/src/raven_toolbox/experiments/conditions.py @@ -0,0 +1,234 @@ +"""Simulate the measured conditions in a model. + +This module only does the FBA half of the distance: given a model and one +:class:`~raven_toolbox.experiments.flux_data.FluxData` dataset (either +``experiment_data.flux_data`` or ``experiment_data.max_growth``), simulate every +condition and report growth and exchange fluxes. +:mod:`.distance` turns those raw numbers into an RMSE against the +matching experimental values -- kept separate so the RMSE math is +testable with hand-built inputs and no model or FBA at all. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Callable, Optional + +import numpy as np + +if TYPE_CHECKING: + import cobra + + from .flux_data import FluxData + + +@dataclass +class ConditionSimResult: + """One condition's simulated outcome. + + ``feasible=False`` (LP infeasible/unbounded, or no biomass flux) + means ``growth``/``exch_fluxes``/``block_fluxes`` are meaningless; + :mod:`.distance` applies the fixed infeasibility penalty for that + row instead of reading them. + + ``skipped=True`` marks a condition with weight 0, which is not + simulated; it is also ``feasible=False``, and :mod:`.distance` leaves + it out of the distance. + """ + + feasible: bool + growth: float = float("nan") + exch_fluxes: dict[str, float] = field(default_factory=dict) + block_fluxes: dict[str, float] = field(default_factory=dict) + skipped: bool = False + + +def simulate_conditions( + model: "cobra.Model", + flux_data: "FluxData", + *, + constrain: bool, + zero_flux_rxns: list[str], + bio_rxn_id: str, + make_anaerobic: Optional[Callable[["cobra.Model"], None]] = None, + change_protein_biomass: Optional[Callable[["cobra.Model", float], None]] = None, + optimize: Optional[Callable[["cobra.Model"], "cobra.Solution"]] = None, +) -> list[ConditionSimResult]: + """Simulate every condition in one dataset. + + The steps are: + + 0. A condition with ``flux_data.weight`` exactly 0 is left out: it is + not matched to an exchange, not simulated, and its result is + ``ConditionSimResult(feasible=False, skipped=True)``. Its name need + not match any exchange metabolite. + 1. Every exchange reaction that appears as a *condition name* + of a condition with a weight other than 0 (matched by name against + ``flux_data.exch_mets``) is blocked (``lower_bound = 0``) up + front, across the board -- not just the reaction for the row + being solved -- so one row's carbon source can't leak flux into + another row's simulation. + 2. For row ``i``, that row's own carbon-source reaction is + unblocked: fixed at the measured uptake rate if ``constrain`` + is True (``experiment_data.flux_data``), or fully opened to ``-1000`` + if ``constrain`` is False (``experiment_data.max_growth`` -- "what's + the *best possible* growth on this carbon source", not "growth + at this measured rate"). + 3. Optional per-condition adjustments run if the corresponding + callable is supplied: anaerobic switch when the oxygen exchange + column reads exactly 0, protein-content rescaling when + ``Ptot`` is given and nonzero. + 4. Solve; record growth, plus (``constrain`` datasets only, since + ``max_growth`` has no exchange measurements to compare against) + every non-NaN measured exchange's simulated flux and every + ``zero_flux_rxns`` entry's simulated flux. + + All mutation happens inside ``with model:``, which cobra reverts + on exit -- only bounds are touched here, so this is safe to call + repeatedly against one persistent (per-worker) model instance; a + copying the model per call is avoided (prohibitively + expensive on a real-scale model). + + Parameters + ---------- + model + The model, with every other constraint already applied. + flux_data + The dataset to simulate. + constrain + True for flux data (fix uptake at the measured rate), False + for max-growth data (open uptake fully). + zero_flux_rxns + Reaction IDs assumed zero-flux in every condition + (``experiment_data.zero_flux``); the row's own carbon source is + skipped even if it's also listed here. + bio_rxn_id + The biomass reaction ID. + make_anaerobic, change_protein_biomass + Optional organism-specific hooks, with no organism-agnostic default. + Data with no anaerobic or ``Ptot`` rows never needs them, and + passing ``None`` for either skips that adjustment instead of + raising. + optimize + Function that solves the model and returns a solution with + ``status``, ``objective_value`` and ``fluxes``. ``None`` calls + ``model.optimize()``. + + Returns + ------- + list[ConditionSimResult] + One entry per row of ``flux_data.conds``, in order. + + Raises + ------ + ValueError + If the name of a condition with a weight other than 0 can't be + matched to an exchange metabolite in ``flux_data.exch_mets``. + """ + if optimize is None: + optimize = _default_optimize + active = _active_conditions(flux_data) + cond_to_exch_idx = _match_conditions_to_exch(flux_data, active) + all_condition_rxn_ids = sorted( + {flux_data.exch_rxn_ids[j] for j in set(cond_to_exch_idx.values())} + ) + o2_col = _find_col(flux_data.exch_mets, "oxygen") + measured_cols = [ + j for j in range(len(flux_data.exch_rxn_ids)) + ] if constrain else [] + + results: list[ConditionSimResult] = [] + for i in range(len(flux_data.conds)): + if i not in cond_to_exch_idx: + results.append(ConditionSimResult(feasible=False, skipped=True)) + continue + exch_idx = cond_to_exch_idx[i] + carbon_rxn_id = flux_data.exch_rxn_ids[exch_idx] + with model: + for rxn_id in all_condition_rxn_ids: + model.reactions.get_by_id(rxn_id).lower_bound = 0.0 + + carbon_rxn = model.reactions.get_by_id(carbon_rxn_id) + if constrain: + carbon_rxn.lower_bound = float(flux_data.exch_fluxes[i, exch_idx]) + else: + carbon_rxn.lower_bound = -1000.0 + + if o2_col is not None and flux_data.exch_fluxes[i, o2_col] == 0.0: + if make_anaerobic is not None: + make_anaerobic(model) + + ptot = float(flux_data.p_tot[i]) + if not (np.isnan(ptot) or ptot == 0.0): + if change_protein_biomass is not None: + change_protein_biomass(model, ptot) + + sol = optimize(model) + if ( + sol.status != "optimal" + or sol.objective_value is None + or np.isnan(sol.objective_value) + ): + results.append(ConditionSimResult(feasible=False)) + continue + + growth = float(sol.fluxes.get(bio_rxn_id, 0.0)) + exch_fluxes: dict[str, float] = {} + block_fluxes: dict[str, float] = {} + if constrain: + for j in measured_cols: + if np.isnan(flux_data.exch_fluxes[i, j]): + continue + rxn_id = flux_data.exch_rxn_ids[j] + exch_fluxes[rxn_id] = float(sol.fluxes.get(rxn_id, 0.0)) + for rxn_id in zero_flux_rxns: + if rxn_id == carbon_rxn_id: + continue + block_fluxes[rxn_id] = float(sol.fluxes.get(rxn_id, 0.0)) + results.append( + ConditionSimResult( + feasible=True, + growth=growth, + exch_fluxes=exch_fluxes, + block_fluxes=block_fluxes, + ) + ) + return results + + +def _default_optimize(model: "cobra.Model") -> "cobra.Solution": + return model.optimize() + + +def _active_conditions(flux_data: "FluxData") -> list[int]: + """Indices of the conditions with a weight other than 0.""" + if flux_data.weight is None: + return list(range(len(flux_data.conds))) + return [i for i in range(len(flux_data.conds)) if flux_data.weight[i] != 0] + + +def _match_conditions_to_exch( + flux_data: "FluxData", active: list[int], +) -> dict[int, int]: + """Column index into ``exch_mets``/``exch_rxn_ids`` for each active + condition, matched by name. Raises ``ValueError`` when a condition + cannot be matched.""" + met_to_idx: dict[str, int] = {} + for j, met in enumerate(flux_data.exch_mets): + met_to_idx.setdefault(met, j) + missing = sorted({flux_data.conds[i] for i in active + if flux_data.conds[i] not in met_to_idx}) + if missing: + raise ValueError( + 'Carbon source(s) "' + + "; ".join(missing) + + '" in the provided fluxData or maxGrowth cannot be matched ' + "by name with an exchange reaction." + ) + return {i: met_to_idx[flux_data.conds[i]] for i in active} + + +def _find_col(exch_mets: list[str], name: str) -> Optional[int]: + try: + return exch_mets.index(name) + except ValueError: + return None diff --git a/src/raven_toolbox/experiments/constrain.py b/src/raven_toolbox/experiments/constrain.py new file mode 100644 index 0000000..1045bcf --- /dev/null +++ b/src/raven_toolbox/experiments/constrain.py @@ -0,0 +1,175 @@ +"""Set model exchange-flux bounds from one condition of a flux dataset. + +In a typical chemostat experiment you measure exchange fluxes +(substrate uptake, byproduct secretion, biomass growth rate) +under one or more growth conditions. :func:`apply_flux_data_constraints` takes a +:class:`~raven_toolbox.experiments.flux_data.FluxData`, selects one condition, +and applies those measurements as bounds on the matching +exchange reactions in the model. + +The ``loose_strict_flux`` argument controls how tightly: ``"loose"`` +gives the model room to wiggle around the measurement; a numeric +percentage clamps it within that band. +""" +from __future__ import annotations + +from typing import TYPE_CHECKING, Literal, Optional, Union + +import numpy as np + +if TYPE_CHECKING: + import cobra + + from .flux_data import FluxData + + +_LooseStrict = Union[Literal["loose"], float] + + +def apply_flux_data_constraints( + model: "cobra.Model", + flux_data: "FluxData", + *, + bio_rxn_id: str, + condition: int | str = 0, + max_min_growth: Literal["max", "min"] = "max", + loose_strict_flux: _LooseStrict = "loose", + c_source: Optional[str] = None, +) -> None: + """Constrain the model's exchange fluxes from a ``FluxData`` measurement. + + Sets the biomass reaction bounds to the measured growth rate and + each exchange reaction in ``flux_data.exch_rxn_ids`` to its + measured flux for the chosen condition. Also zeros out the + preferred carbon source so the data determines the choice. + + A ``+/-1000`` measured flux means unconstrained, regardless of + ``loose_strict_flux``: + ``-1000`` -> ``lb=-1000, ub=0`` (free uptake), ``+1000`` -> + ``lb=0, ub=1000`` (free excretion). + + Parameters + ---------- + model + Model with the exchange reactions referenced by + ``flux_data`` already in place. Mutated in place. + flux_data + Flux data, for example from + :func:`~raven_toolbox.experiments.flux_data.load_flux_data`. + bio_rxn_id + Biomass reaction ID. + condition + Either the 0-indexed condition number, or the condition name + (matched against ``flux_data.conds``). + max_min_growth + ``"max"`` sets ``bio_rxn_id`` upper bound to the measured + growth rate (and lower bound to 0); ``"min"`` sets the lower + bound (suitable when minimising a resource such as the protein + pool). + loose_strict_flux + ``"loose"`` keeps one bound at 0 and the other at the + measured value (allows zero flux). A numeric percentage + ``p`` brackets the measured value with hard bounds: + ``lb = val * (1 - p/200)``, ``ub = val * (1 + p/200)``, + swapped as needed to keep ``lb <= ub`` when ``val`` is + negative. ``p=10`` allows 10% total variance (i.e. +/-5%). + c_source + Reaction ID of the preferred carbon-source uptake reaction; + its bounds are set to ``(0, 0)`` before applying the flux + data so that the data alone determines the carbon source + used. ``None`` or ``""`` blocks no reaction. Silently ignored + if the reaction id is not found in the model. + + Raises + ------ + ValueError + If ``max_min_growth`` / ``loose_strict_flux`` is invalid, + ``condition`` (as string) is not in ``flux_data.conds``, or any + ``flux_data.exch_rxn_ids`` is missing from the model. + IndexError + If ``condition`` (as int) is out of range. + """ + if max_min_growth not in ("max", "min"): + raise ValueError( + f"max_min_growth must be 'max' or 'min', got {max_min_growth!r}" + ) + if not ( + loose_strict_flux == "loose" + or (isinstance(loose_strict_flux, (int, float)) + and not isinstance(loose_strict_flux, bool) + and 0 <= loose_strict_flux <= 100) + ): + raise ValueError( + f"loose_strict_flux must be 'loose' or a number in [0, 100]; " + f"got {loose_strict_flux!r}" + ) + + if isinstance(condition, str): + if condition not in flux_data.conds: + raise ValueError( + f"Condition {condition!r} not found in flux_data.conds " + f"({flux_data.conds})" + ) + cond_idx = flux_data.conds.index(condition) + else: + if not 0 <= condition < len(flux_data.conds): + raise IndexError( + f"condition index {condition} out of range " + f"(have {len(flux_data.conds)} condition(s))" + ) + cond_idx = condition + + fluxes = flux_data.exch_fluxes[cond_idx, :] + cobra_rxn_ids = {r.id for r in model.reactions} + missing = [r for r in flux_data.exch_rxn_ids if r not in cobra_rxn_ids] + if missing: + preview = missing[:5] + raise ValueError( + f"{len(missing)} exchange reaction ID(s) from flux_data are not " + f"present in the model (examples: {preview})" + ) + + if c_source: + try: + c_source_rxn = model.reactions.get_by_id(c_source) + c_source_rxn.lower_bound = 0.0 + c_source_rxn.upper_bound = 0.0 + except KeyError: + pass # c_source not in model; silent + + bio_rxn = model.reactions.get_by_id(bio_rxn_id) + gr = float(flux_data.gr_rate[cond_idx]) + if max_min_growth == "max": + bio_rxn.lower_bound = 0.0 + bio_rxn.upper_bound = gr + else: + bio_rxn.lower_bound = gr + bio_rxn.upper_bound = 1000.0 + + for rxn_id, flux in zip(flux_data.exch_rxn_ids, fluxes): + if np.isnan(flux): + continue + rxn = model.reactions.get_by_id(rxn_id) + + if abs(flux) == 1000.0: + if flux == -1000.0: + rxn.lower_bound = -1000.0 + rxn.upper_bound = 0.0 + else: + rxn.lower_bound = 0.0 + rxn.upper_bound = 1000.0 + continue + + if loose_strict_flux == "loose": + if flux < 0: + rxn.lower_bound = float(flux) + rxn.upper_bound = 0.0 + else: + rxn.lower_bound = 0.0 + rxn.upper_bound = float(flux) + else: + pct = float(loose_strict_flux) + lo = flux * (1 - pct / 200.0) + hi = flux * (1 + pct / 200.0) + rxn.lower_bound = float(min(lo, hi)) + rxn.upper_bound = float(max(lo, hi)) diff --git a/src/raven_toolbox/experiments/data.py b/src/raven_toolbox/experiments/data.py new file mode 100644 index 0000000..2484f1e --- /dev/null +++ b/src/raven_toolbox/experiments/data.py @@ -0,0 +1,88 @@ +"""Load the measured growth rates and fluxes of the conditions. + +The flux and max-growth files, ``fluxData.tsv`` and ``maxGrowth.tsv``, +share the column layout +``Condition Ptot grRate () ... weight source``, so both use +:func:`~raven_toolbox.experiments.flux_data.load_flux_data`; only ``zeroFlux.tsv`` +(a single ``Rxns`` id column) needs a loader of its own. A ``weight`` of 0 leaves a +condition out of the simulation and of the distance to the measurements. + +All three files are optional: a missing file yields ``None`` (or an empty +list for ``zero_flux``), never an exception. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +from .flux_data import FluxData, load_flux_data + +_FLUX_DATA_FILENAME = "fluxData.tsv" +_MAX_GROWTH_FILENAME = "maxGrowth.tsv" +_ZERO_EXCH_FILENAME = "zeroFlux.tsv" + + +@dataclass +class ExperimentData: + """Measured conditions. + + Attributes + ---------- + flux_data + Per-condition growth/exchange-flux measurements, from + ``fluxData.tsv``. ``None`` if that file isn't present. + max_growth + Per-condition maximum-growth-rate measurements (one active + carbon source per condition, at ``-1000`` i.e. unconstrained), + from ``maxGrowth.tsv``. ``None`` if that file isn't + present. + zero_flux + Reaction IDs assumed to carry zero flux, from + ``zeroFlux.tsv``. Empty if that file isn't present. + """ + + flux_data: Optional[FluxData] = None + max_growth: Optional[FluxData] = None + zero_flux: list[str] = field(default_factory=list) + + +def load_experiment_data(data_dir: str | Path) -> ExperimentData: + """Load the three experimental data files. + + Each file is optional and resolved as + ``/``; a missing file yields + ``None`` (``flux_data``/``max_growth``) or an empty list + (``zero_flux``) rather than raising. + + Parameters + ---------- + data_dir + The folder that holds the files. + + Returns + ------- + ExperimentData + """ + data_dir = Path(data_dir) + return ExperimentData( + flux_data=_load_optional_flux_data(data_dir / _FLUX_DATA_FILENAME), + max_growth=_load_optional_flux_data(data_dir / _MAX_GROWTH_FILENAME), + zero_flux=_load_zero_exch(data_dir / _ZERO_EXCH_FILENAME), + ) + + +def _load_optional_flux_data(path: Path) -> Optional[FluxData]: + if not path.is_file(): + return None + return load_flux_data(path) + + +def _load_zero_exch(path: Path) -> list[str]: + """Parse ``zeroFlux.tsv``: a single ``Rxns`` header, then + one reaction ID per line.""" + if not path.is_file(): + return [] + lines = path.read_text(encoding="utf-8").splitlines() + rxn_ids = [line.strip() for line in lines[1:] if line.strip()] + return rxn_ids diff --git a/src/raven_toolbox/experiments/distance.py b/src/raven_toolbox/experiments/distance.py new file mode 100644 index 0000000..7c00de1 --- /dev/null +++ b/src/raven_toolbox/experiments/distance.py @@ -0,0 +1,264 @@ +"""Carbon- and condition-weighted RMSE distance between a model and the data. + +:mod:`.conditions` does the FBA half; this module is pure numpy (plus one +small model-reading helper for the carbon weights) so the RMSE math is +directly testable against hand-computed expected values, with no model or +FBA involved. + +The biomass reaction is passed explicitly as ``bio_rxn_id`` +rather than matched by a literal name, and +:func:`dataset_rmse` builds its reaction-id-to-column lookup once. + +Every zero-carbon exchange metabolite (O2, water, ...) is clamped up to +a carbon weight of 1, not 0, so those reactions' flux mismatches still +count toward the RMSE at unit weight rather than being zeroed out of it. +See :func:`compute_carbon_weights`. +""" +from __future__ import annotations + +import warnings +from typing import TYPE_CHECKING, Iterable + +import numpy as np + +from .conditions import ConditionSimResult + +if TYPE_CHECKING: + import cobra + + from .data import ExperimentData + from .flux_data import FluxData + +#: Assumed carbon content of the biomass reaction, in Cmmol/gDCW, used to put growth-rate deviations on the +#: same carbon-weighted scale as exchange-flux deviations. +BIOMASS_CARBON_EQUIV = 41.0 + +#: Fixed penalty substituted for an infeasible/failed condition's RMSE +#: ("clearly worse than any real RMSE": it must dominate the mean). +INFEASIBLE_PENALTY = 99.0 + + +def compute_carbon_weights( + model: "cobra.Model", rxn_ids: Iterable[str], bio_rxn_id: str, +) -> dict[str, float]: + """Per-reaction carbon-weight lookup for RMSE weighting. + + The biomass reaction gets :data:`BIOMASS_CARBON_EQUIV`. Every + other reaction gets the carbon count of its single exchanged + metabolite (via cobrapy's parsed ``Metabolite.elements``, not a + hand-rolled formula parser). A missing/unparseable formula, or a + computed carbon count of exactly 0 (e.g. O2, water), both fall + back to 1. A model in which no + metabolite has a formula gives every exchange the weight 1 and + emits a warning. + + Parameters + ---------- + model + Model providing the metabolite formulas. + rxn_ids + Reaction IDs to compute weights for (typically every + reaction referenced by a dataset: its exchange + columns, its zero-flux list, and the biomass reaction). + bio_rxn_id + The biomass reaction ID. + + Returns + ------- + dict[str, float] + """ + carbon_weights: dict[str, float] = {} + if not any(m.formula for m in model.metabolites): + warnings.warn( + "The model has no metabolite formulas; every exchange " + "reaction gets the carbon weight 1.", + UserWarning, stacklevel=2, + ) + for rxn_id in rxn_ids: + if rxn_id == bio_rxn_id: + carbon_weights[rxn_id] = BIOMASS_CARBON_EQUIV + continue + rxn = model.reactions.get_by_id(rxn_id) + mets = list(rxn.metabolites) + carbon = mets[0].elements.get("C") if len(mets) == 1 else None + value = float(carbon) if carbon is not None else 1.0 + carbon_weights[rxn_id] = value if value != 0.0 else 1.0 + return carbon_weights + + +def dataset_rmse( + flux_data: "FluxData", + sims: list[ConditionSimResult], + *, + constrain: bool, + carbon_weights: dict[str, float], + bio_rxn_id: str, + penalty: float = INFEASIBLE_PENALTY, +) -> tuple[float, np.ndarray]: + """RMSE (mean, per-condition array) for one dataset. + + A condition with ``flux_data.weight`` exactly 0 is left out: its entry in + the per-condition array is ``NaN`` and it is not part of the mean. + + Parameters + ---------- + flux_data + The dataset being scored (``experiment_data.flux_data`` or + ``experiment_data.max_growth``). + sims + One :class:`~.conditions.ConditionSimResult` per + ``flux_data.conds`` row, e.g. from + :func:`~.conditions.simulate_conditions`. The entries of the + conditions with weight 0 are not read. + constrain + Must match the value passed to + :func:`~.conditions.simulate_conditions` for these + ``sims``: True includes carbon-weighted exchange-flux terms + (flux data), False scores growth alone (max-growth data). + carbon_weights + Carbon-weight lookup from :func:`compute_carbon_weights`, covering + every reaction referenced by ``sims``' ``exch_fluxes`` / + ``block_fluxes`` plus ``bio_rxn_id``. A reaction missing from + the lookup falls back to a weight of 1. + bio_rxn_id + The biomass reaction ID (for reading ``flux_data.gr_rate``'s + carbon-equivalent counterpart). + penalty + RMSE substituted for an infeasible condition. + + Returns + ------- + rmse : float + Weighted (by ``flux_data.weight``, if present) + mean of the per-condition RMSEs of the conditions with a weight + other than 0. ``NaN`` if every condition has weight 0. + rmse_list : numpy.ndarray + The per-condition RMSEs (post-penalty, pre-weighting), shape + ``(len(flux_data.conds),)``; ``NaN`` for a condition with weight 0. + """ + n = len(flux_data.conds) + if len(sims) != n: + raise ValueError( + f"sims has {len(sims)} entries; expected {n} to match " + f"flux_data.conds." + ) + rxn_to_col = {r: j for j, r in enumerate(flux_data.exch_rxn_ids)} + + if flux_data.weight is None: + active = np.ones(n, dtype=bool) + else: + active = np.asarray(flux_data.weight) != 0 + + rmse_list = np.empty(n, dtype=float) + for i, sim in enumerate(sims): + if not active[i]: + rmse_list[i] = np.nan + continue + if not sim.feasible: + rmse_list[i] = penalty + continue + + bio_meas = float(flux_data.gr_rate[i]) * BIOMASS_CARBON_EQUIV + bio_sim = sim.growth * BIOMASS_CARBON_EQUIV + + if not constrain: + rmse_list[i] = abs(bio_meas - bio_sim) + continue + + measured = [bio_meas] + simulated = [bio_sim] + for rxn_id, sim_flux in sim.exch_fluxes.items(): + w = carbon_weights.get(rxn_id, 1.0) + measured.append(w * float(flux_data.exch_fluxes[i, rxn_to_col[rxn_id]])) + simulated.append(w * sim_flux) + for rxn_id, sim_flux in sim.block_fluxes.items(): + w = carbon_weights.get(rxn_id, 1.0) + weighted_sim = w * sim_flux + if weighted_sim == 0.0: + # Already-correct zero predictions don't dilute the RMSE + continue + measured.append(0.0) + simulated.append(weighted_sim) + + diff = np.asarray(measured, dtype=float) - np.asarray(simulated, dtype=float) + rmse_list[i] = float(np.sqrt(np.mean(diff ** 2))) + + if not active.any(): + return float("nan"), rmse_list + weighted = rmse_list + if flux_data.weight is not None: + weighted = rmse_list * flux_data.weight + return float(np.mean(weighted[active])), rmse_list + + +def experiment_distance( + experiment_data: "ExperimentData", + *, + flux_sims: list[ConditionSimResult] | None, + max_growth_sims: list[ConditionSimResult] | None, + carbon_weights: dict[str, float], + bio_rxn_id: str, + penalty: float = INFEASIBLE_PENALTY, + max_growth_weight: float = 1.0, +) -> tuple[float, dict[str, np.ndarray]]: + """Combine both datasets' RMSE into one score. + + ``flux_sims`` must be given (non-``None``) iff ``experiment_data.flux_data`` + is not ``None``, and likewise for ``max_growth_sims`` / + ``experiment_data.max_growth``. A dataset absent from ``experiment_data`` doesn't + contribute to the combined RMSE, and neither does a dataset in which + every condition has weight 0. + + Parameters + ---------- + max_growth_weight + Relative weight of the max-growth dataset against the flux + dataset, giving ``(rmse_flux + w * rmse_max_growth) / (w + 1)``. + At 2 the max-growth dataset counts twice as much as the flux + dataset in the combination. + + Returns + ------- + rmse : float + Weighted mean of whichever dataset RMSEs are present. ``NaN`` + if neither dataset is present or no condition of the present + datasets has a weight other than 0. + detail : dict[str, numpy.ndarray] + Per-condition RMSE arrays, keyed ``"flux_data"`` / + ``"max_growth"`` for whichever datasets were scored. + """ + parts: list[float] = [] + weights: list[float] = [] + detail: dict[str, np.ndarray] = {} + + if experiment_data.flux_data is not None: + if flux_sims is None: + raise ValueError("experiment_data.flux_data is set but flux_sims is None.") + rmse, per_cond = dataset_rmse( + experiment_data.flux_data, flux_sims, + constrain=True, carbon_weights=carbon_weights, bio_rxn_id=bio_rxn_id, + penalty=penalty, + ) + if not np.isnan(rmse): + parts.append(rmse) + weights.append(1.0) + detail["flux_data"] = per_cond + + if experiment_data.max_growth is not None: + if max_growth_sims is None: + raise ValueError("experiment_data.max_growth is set but max_growth_sims is None.") + rmse, per_cond = dataset_rmse( + experiment_data.max_growth, max_growth_sims, + constrain=False, carbon_weights=carbon_weights, bio_rxn_id=bio_rxn_id, + penalty=penalty, + ) + if not np.isnan(rmse): + parts.append(rmse) + weights.append(float(max_growth_weight)) + detail["max_growth"] = per_cond + + if parts: + rmse = float(np.dot(parts, weights) / np.sum(weights)) + else: + rmse = float("nan") + return rmse, detail diff --git a/src/raven_toolbox/experiments/flux_data.py b/src/raven_toolbox/experiments/flux_data.py new file mode 100644 index 0000000..8cd6685 --- /dev/null +++ b/src/raven_toolbox/experiments/flux_data.py @@ -0,0 +1,258 @@ +"""Container for the measured growth rates and exchange fluxes of several +conditions, and the parser of the ``fluxData.tsv`` format. +""" +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +import numpy as np + +logger = logging.getLogger(__name__) + + +# ` ()` -> capture met_name and rxn_id. + + +@dataclass +class FluxData: + """Per-condition flux measurements with per-reaction exchange data. + + Attributes + ---------- + conds + Names of the experimental conditions / samples. + p_tot + Total protein content per condition, g/gDCW. Shape + ``(n_conds,)``. + gr_rate + Measured growth rate per condition, 1/h. Shape ``(n_conds,)``. + exch_fluxes + Exchange flux per (condition, reaction), mmol/gDCW/h. Shape + ``(n_conds, n_rxns)``. NaN means "no measurement"; ``+/-1000`` + means unconstrained (used by + ``apply_flux_data_constraints``). + exch_mets + Metabolite names matching the columns of ``exch_fluxes``. + exch_rxn_ids + Exchange reaction IDs in the model, matching the columns of + ``exch_fluxes`` and ``exch_mets``. + weight + Optional per-condition weights of the distance to the + measurements. A weight of 0 leaves the condition out of the + simulation and of the distance. ``None`` if the source file did not + have the ``weight`` column. + source + Optional per-condition free-text describing where the data + came from. ``None`` if the source file did not have a + ``source`` column. + """ + + conds: list[str] = field(default_factory=list) + p_tot: np.ndarray = field( + default_factory=lambda: np.empty(0, dtype=float) + ) + gr_rate: np.ndarray = field( + default_factory=lambda: np.empty(0, dtype=float) + ) + exch_fluxes: np.ndarray = field( + default_factory=lambda: np.empty((0, 0), dtype=float) + ) + exch_mets: list[str] = field(default_factory=list) + exch_rxn_ids: list[str] = field(default_factory=list) + weight: Optional[np.ndarray] = None + source: Optional[list[str]] = None + + +def load_flux_data(path: str | Path) -> FluxData: + """Parse a ``fluxData.tsv`` into a ``FluxData``. + + File format (tab-delimited, one header row): + + condition Ptot grRate () () ... + c1 0.5 0.4 -10.0 5.0 ... + c2 0.6 0.5 -8.5 4.2 ... + + Optional extra columns (in any position): + + * ``weight`` (float): weight of the condition in the distance to the measurements; 0 + leaves the condition out of the simulation and of the distance. + * ``source`` (str): free-text describing the data origin. + + An exchange column header has the form `` ()``, split at + the parenthesis that matches the final closing one, so a reaction id + may contain parentheses (``X (EX_a(e))`` gives ``X`` and + ``EX_a(e)``). + + Both columns are dropped from the matrix and surfaced on the + returned ``FluxData`` as ``weight`` and ``source``. + + Parameters + ---------- + path + Path to the TSV file. + + Returns + ------- + FluxData + + Raises + ------ + FileNotFoundError + If ``path`` does not exist. + ValueError + If the file has no header row, no exchange columns, or a row + with a different number of cells than the header. + """ + path = Path(path) + if not path.is_file(): + raise FileNotFoundError(f"fluxData file not found: {path}") + + rows: list[list[str]] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + rows.append(line.rstrip("\n").split("\t")) + + if not rows: + raise ValueError(f"{path} is empty.") + if len(rows[0]) < 3: + raise ValueError( + f"{path}: header has only {len(rows[0])} column(s); " + f"expected at least 'condition Ptot grRate'." + ) + + header = rows[0] + data = rows[1:] + for line_no, r in enumerate(data, start=2): + if len(r) != len(header): + raise ValueError( + f"{path}: row {line_no} has {len(r)} cell(s), the header " + f"has {len(header)}." + ) + + # Pull out optional columns first; collect indices to drop. + optional_indices: list[int] = [] + if _index_of(header, "evotuneRMSEweight") is not None: + raise ValueError( + f"{path}: the column 'evotuneRMSEweight' is now called 'weight'." + ) + weight_idx = _index_of(header, "weight") + source_idx = _index_of(header, "source") + + weights: Optional[np.ndarray] = None + source_values: Optional[list[str]] = None + + if weight_idx is not None: + weights = np.array( + [_parse_float(r[weight_idx]) for r in data], + dtype=float, + ) + optional_indices.append(weight_idx) + if source_idx is not None: + source_values = [r[source_idx] for r in data] + optional_indices.append(source_idx) + + # Build the kept-column index list (preserving order). + kept = [j for j in range(len(header)) if j not in optional_indices] + kept_header = [header[j] for j in kept] + + # First three columns are condition / Ptot / grRate. Remaining are exchanges. + if len(kept_header) < 4: + raise ValueError( + f"{path}: no exchange-flux columns found after the first three." + ) + + exch_headers = kept_header[3:] + exch_mets, exch_rxn_ids = _split_exch_headers(exch_headers) + + conds = [r[kept[0]] for r in data] + p_tot = np.array([_parse_float(r[kept[1]]) for r in data], dtype=float) + gr_rate = np.array([_parse_float(r[kept[2]]) for r in data], dtype=float) + exch_fluxes = np.full((len(data), len(exch_headers)), np.nan, dtype=float) + for i, r in enumerate(data): + for ci, j in enumerate(kept[3:]): + exch_fluxes[i, ci] = _parse_float(r[j]) + + return FluxData( + conds=conds, + p_tot=p_tot, + gr_rate=gr_rate, + exch_fluxes=exch_fluxes, + exch_mets=exch_mets, + exch_rxn_ids=exch_rxn_ids, + weight=weights, + source=source_values, + ) + + +def _index_of(header: list[str], name: str) -> Optional[int]: + """Return the index of `name` in `header`, or None if absent.""" + for j, h in enumerate(header): + if h.strip() == name: + return j + return None + + +def _parse_float(s: str) -> float: + """Parse a numeric cell; empty / non-numeric -> NaN.""" + s = s.strip() + if not s: + return float("nan") + try: + return float(s) + except ValueError: + return float("nan") + + +def _split_header(header: str) -> Optional[tuple[str, str]]: + """Split ``' ()'`` at the parenthesis that matches the final + closing one; return None for a header of another form.""" + if not header.endswith(")"): + return None + depth = 0 + for k in range(len(header) - 1, -1, -1): + if header[k] == ")": + depth += 1 + elif header[k] == "(": + depth -= 1 + if depth == 0: + rxn = header[k + 1:-1].strip() + if not rxn: + return None + return header[:k].strip(), rxn + return None + + +def _split_exch_headers(headers: list[str]) -> tuple[list[str], list[str]]: + """Split each header `' ()'` into (met, rxn) pairs. + + Headers without parentheses pass through with the rxn_id equal + to the full header (the + capture-group falls through unchanged). + """ + mets: list[str] = [] + rxns: list[str] = [] + no_paren: list[str] = [] + for h in headers: + parts = _split_header(h.strip()) + if parts is not None: + mets.append(parts[0]) + rxns.append(parts[1]) + else: + mets.append(h.strip()) + rxns.append(h.strip()) + no_paren.append(h.strip()) + if no_paren: + # The reaction id is taken from the header, so a column without the + # ` ()` form will be matched against a reaction whose id is + # the metabolite name -- almost certainly wrong. Surface it. + logger.warning( + "load_flux_data: %d exchange column header(s) lack the " + "' ()' form; their reaction id falls back to the full " + "header text: %s", + len(no_paren), no_paren[:5], + ) + return mets, rxns diff --git a/src/raven_toolbox/utils/protein.py b/src/raven_toolbox/utils/protein.py new file mode 100644 index 0000000..a87ea97 --- /dev/null +++ b/src/raven_toolbox/utils/protein.py @@ -0,0 +1,113 @@ +"""Compute the molecular weight of a protein from its amino acid sequence.""" +from __future__ import annotations + +import logging + +__all__ = ["calculate_mw"] + +logger = logging.getLogger(__name__) + + +# Average mass of water (g/mol). Added once to account for the two +# H atoms and one OH that remain after peptide bond condensation. +_WATER_MASS = 18.01528 + + +# Average residue masses (g/mol). Standard amino acids agree with +# widely-cited references (e.g., ExPASy ProtParam). Non-standard codes: +# - B = average(D, N). +# - J = same as L and I, which both happen to equal 113.16. +# - O (pyrrolysine) = 255.31. +# - U (selenocysteine) = 150.04. +# - Z = average(E, Q). +# +# X (any amino acid): the unweighted mean of the 20 standard residues +# above, 118.885. +_STANDARD_RESIDUE_MASSES: dict[str, float] = { + "A": 71.08, + "C": 103.14, + "D": 115.09, + "E": 129.11, + "F": 147.17, + "G": 57.05, + "H": 137.14, + "I": 113.16, + "K": 128.17, + "L": 113.16, + "M": 131.20, + "N": 114.10, + "P": 97.12, + "Q": 128.13, + "R": 156.19, + "S": 87.08, + "T": 101.10, + "V": 99.13, + "W": 186.21, + "Y": 163.17, +} + +_X_MASS = sum(_STANDARD_RESIDUE_MASSES.values()) / len(_STANDARD_RESIDUE_MASSES) + +_RESIDUE_MASSES: dict[str, float] = { + **_STANDARD_RESIDUE_MASSES, + "B": (115.09 + 114.10) / 2, # D-or-N + "J": 113.16, # L-or-I (both 113.16) + "O": 255.31, # pyrrolysine + "U": 150.04, # selenocysteine + "X": _X_MASS, # any standard residue + "Z": (129.11 + 128.13) / 2, # E-or-Q +} + + +def calculate_mw(sequence: str) -> float: + """Compute the molecular weight of a protein from its sequence. + + Each residue contributes its average mass; one water mass is added + once (representing the H + OH at the peptide termini, equivalent to + one un-eliminated water relative to N residues + N water masses). + + Recognized amino acid codes (the sequence is uppercased first, so + matching is case-insensitive): + + ACDEFGHIKLMNPQRSTVWY - the 20 standard residues + BJOZUX - non-standard / ambiguous codes + + Whitespace and digits are skipped silently. Any other unrecognized + character is also skipped, but triggers one warning per call + listing every distinct offending character. + + Parameters + ---------- + sequence + Protein sequence as a string. May contain whitespace, digits, + and other non-letter characters; those are ignored. + + Returns + ------- + float + Molecular weight in g/mol (Da). Returns ``nan`` for an + empty/all-skipped sequence (no residues), so a missing sequence + is not mistaken for an 18 Da "protein". + """ + upper = sequence.upper() if sequence else "" + mw = _WATER_MASS + n_residues = 0 + + unknown_chars: set[str] = set() + for ch in upper: + if ch in _RESIDUE_MASSES: + mw += _RESIDUE_MASSES[ch] + n_residues += 1 + elif not ch.isspace() and not ch.isdigit(): + unknown_chars.add(ch) + + if unknown_chars: + logger.warning( + "calculate_mw: skipped %d unknown character(s): %s", + len(unknown_chars), + ", ".join(repr(c) for c in sorted(unknown_chars)), + ) + + if n_residues == 0: + return float("nan") + return mw diff --git a/src/raven_toolbox/utils/solver.py b/src/raven_toolbox/utils/solver.py index 245f1d9..b44618c 100644 --- a/src/raven_toolbox/utils/solver.py +++ b/src/raven_toolbox/utils/solver.py @@ -1,7 +1,86 @@ -"""Solver settings that RAVEN applies to every LP it solves.""" +"""Solver settings and support for optlang's scipy (HiGHS) solver interface. + +:data:`RAVEN_TOLERANCE` and :func:`set_raven_tolerance` give a model the tolerance that +RAVEN gives every LP it solves. + +The scipy interface of optlang differs from the other interfaces in ways that matter for a +large model, and the rest of this module works around them. + +Index map after a removal + ``scipy_interface.Problem`` keeps a map from variable name to column. After a variable is + removed it renumbers the columns with ``islice(variables, start, len(constraints))``, so + the last ``len(variables) - len(constraints)`` names keep their old column whenever the + problem has more variables than constraints. Primal values, and so every flux that cobra + reads, then come from the wrong columns or raise an ``IndexError``, and a later removal + of such a variable deletes the wrong column of the coefficient matrix. Variables are + removed lazily, by the next ``solver.update()``, which any ``add`` or ``optimize`` runs, so + the damage shows up in code that never removed anything itself. + + :func:`repair_scipy_problem` moves the problem to a subclass that renumbers correctly, so + every removal after it is right, and rebuilds the coefficient matrix from the constraint + expressions if the map was already stale. A model class that calls it whenever + ``model.solver`` is read is covered whatever removes the variables: the library, cobra, or + user code that calls ``model.remove_reactions``, ``reaction.delete`` or + ``model.remove_cons_vars``. A model that is built on a different interface and converted + afterwards with :func:`switch_to_scipy` has no removal on an unrepaired scipy problem. + For a plain ``cobra.Model`` on the scipy solver that you edit yourself, call ``repair_scipy_problem(model)`` before the first removal; a + call after the removals still restores the map and the matrix, but anything read in + between was wrong. + +Coefficient matrix + optlang stores the matrix densely and copies it on every added or removed variable or + constraint, so building or editing a genome-scale model (thousands of reactions) takes + hours. The subclass installed by :func:`repair_scipy_problem` keeps the coefficients in + dictionaries and hands HiGHS a sparse matrix, built once per structural change. + ``problem.A`` is then a ``scipy.sparse.csr_matrix`` (a dense array when there are no rows). + +Solution after a solve + ``cobra.Model.optimize`` sets the objective direction again after the solve, and the scipy + interface discards the solution on every direction assignment, also one that changes + nothing. ``Reaction.flux`` and ``Solver.primal_values`` then find no values after + ``model.optimize()``. The subclass discards the solution only when the direction changes. + + After a solve that finds no optimum (infeasible, unbounded) the scipy interface returns + ``None`` for every primal value and raises for the objective, so ``cobra.Model.optimize`` + fails with a ``TypeError`` instead of returning a solution with that status. The subclass + returns NaN for both, so the solution has the status and NaN values. + +Duals + The scipy interface raises for ``Constraint.dual`` and ``Variable.dual`` but keeps the + whole ``linprog`` result in ``problem._solution``. :func:`constraint_shadow_prices` reads + the row marginals from it, so the shadow prices of a scipy model equal those of the glpk + and gurobi interfaces. The stored result is used only after it is shown to be an optimal + primal and dual solution of the problem as it is now, so a result left over from before + a bound or an objective changed is never read. +""" from __future__ import annotations +import contextlib +from itertools import islice +from typing import Iterable + import cobra +import numpy as np +from scipy import sparse + +try: + from optlang import scipy_interface as _scipy_interface + from optlang.util import SolverTolerances + from scipy.optimize import linprog +except ImportError: # pragma: no cover - optlang always includes the interface + _scipy_interface = None + +__all__ = [ + "RAVEN_TOLERANCE", + "set_raven_tolerance", + "repair_scipy_problem", + "switch_to_scipy", + "fast_solver_while_building", + "edit_on_fast_solver", + "is_scipy_interface", + "constraint_shadow_prices", + "optimize_model", +] #: Feasibility and optimality tolerance that RAVEN's ``solveLP`` gives the #: solver (``optimizeProb``). cobrapy leaves the solver's own default, 1e-6 for @@ -18,3 +97,600 @@ def set_raven_tolerance(model: cobra.Model, tolerance: float = RAVEN_TOLERANCE) """ model.tolerance = tolerance return model + + +def optimize_model(model, **kwargs): + """``model.optimize()`` that returns the solution of an infeasible LP on every solver. + + Gurobi holds no primal values for an LP that it presolved as infeasible and optlang's + scipy interface returns ``None`` for them, and cobra reads them while it builds the + solution, so ``model.optimize()`` raises ``GurobiError`` or ``TypeError`` there. When + the solver status is not optimal this function returns a ``Solution`` with that status, + a NaN objective and NaN fluxes instead; with an optimal status every error is raised. + """ + try: + return model.optimize(**kwargs) + except Exception: + if model.solver.status == "optimal": + raise + import pandas as pd + from cobra.core import Solution + + return Solution( + objective_value=float("nan"), + status=model.solver.status, + fluxes=pd.Series(float("nan"), index=[r.id for r in model.reactions]), + ) + + +# Relative tolerance of the optimality certificate, scaled by the size of the quantity checked. +_CERTIFICATE_TOL = 1e-6 + + +if _scipy_interface is not None: + + class _SparseProblem(_scipy_interface.Problem): + """``scipy_interface.Problem`` with a sparse coefficient matrix. + + Renumbers every variable column after a removal and keeps its solution when the + direction is set to the value it has. + """ + + def __init__(self): + super().__init__() + self._init_sparse() + + # Feasibility and optimality tolerance of HiGHS; None keeps its default (1e-7). + _feasibility = None + _optimality = None + + def optimize(self, method="highs", verbosity=False, tolerance=1e-9, **kwargs): + """``linprog`` on the problem, with the tolerances that were set on the model.""" + c = np.array([self.objective.get(name, 0) for name in self._variables], dtype=float) + if self.direction == "max": + c *= -1 + options = {"maxiter": 10000, "disp": verbosity} + bounds = list(self.bounds.values()) + + def solve(options): + return linprog(c, self.A, self.upper_bounds, bounds=bounds, method=method, options=options, **kwargs) + + tight = dict(options) + if self._feasibility is not None: + tight["primal_feasibility_tolerance"] = self._feasibility + if self._optimality is not None: + tight["dual_feasibility_tolerance"] = self._optimality + solution = solve(tight) + # HiGHS reports numerical difficulties (status 4) on some LPs at a tolerance + # below its default; those are solved again at the default. + if solution.status == 4 and tight != options: + solution = solve(options) + self._solution = solution + self._status = solution.status + if self._status == 0: + self._var_primals = solution.x + self._slacks = solution.slack + else: + self._var_primals = None + self._slacks = None + self._f = solution.fun + + @property + def status(self): + """The solver status; a solve that ended with numerical difficulties is ``numeric``.""" + if self._status is None: + raise RuntimeError("Problem has not been optimized yet") + return _scipy_interface.SCIPY_STATUS.get(self._status, _scipy_interface.interface.NUMERIC) + + def _init_sparse(self): + # _rows[row name] = {variable name: coefficient}; _cols[variable name] = {row names} + self._rows = {} + self._cols = {name: set() for name in self._variables} + self._matrix = None + self._A = None + + @property + def direction(self): + return self._direction + + @direction.setter + def direction(self, value): + if value != self._direction: + self._direction = value + self._reset_solution() + + @property + def A(self): + """The linear coefficient matrix: sparse, or a dense array without rows.""" + n, m = len(self._variables), len(self._constraints) + if m == 0: + return np.zeros((0, n)) + if self._matrix is None: + data, rows, cols = [], [], [] + columns = self._variables + for row_name, row_index in self._constraints.items(): + for name, coefficient in self._rows[row_name].items(): + data.append(coefficient) + rows.append(row_index) + cols.append(columns[name]) + self._matrix = sparse.csr_matrix((data, (rows, cols)), shape=(m, n), dtype=float) + return self._matrix + + def get_var_primal(self, name): + """The primal value; NaN after a solve that found no optimum, None before any solve.""" + if self._var_primals is None: + return None if self._status is None else float("nan") + return self._var_primals[self._get_var_index(name)] + + @property + def objective_value(self): + """The optimum; NaN after a solve that found none.""" + if self._f is None and self._status is not None: + return float("nan") + return super().objective_value + + def add_variable(self, name): + if name in self._variables: + raise ValueError("A variable named " + name + " already exists.") + self._variables[name] = len(self._variables) + self.bounds[name] = (0, None) + self._cols[name] = set() + self._matrix = None + self._reset_solution() + + def add_constraint(self, name, coefficients={}, ub=0): + if name in self._constraints: + raise ValueError("A constraint named " + name + " already exists.") + self._constraints[name] = len(self._constraints) + self.upper_bounds = np.append(self.upper_bounds, ub) + row = {v: c for v, c in coefficients.items() if c != 0 and v in self._variables} + self._rows[name] = row + for v in row: + self._cols[v].add(name) + self._matrix = None + self._reset_solution() + + def remove_variable(self, name): + index = self._get_var_index(name) + for row_name in self._cols.pop(name): + del self._rows[row_name][name] + del self.bounds[name] + del self._variables[name] + self._update_variable_indices(index) + self._matrix = None + self._reset_solution() + + def remove_constraint(self, name): + index = self._get_constraint_index(name) + self.upper_bounds = np.delete(self.upper_bounds, index) + for v in self._rows.pop(name): + self._cols[v].discard(name) + del self._constraints[name] + self._update_constraint_indices(index) + self._matrix = None + self._reset_solution() + + def change_var_name(self, name, value): + super().change_var_name(name, value) + if name != value: + for row_name in self._cols[name]: + self._rows[row_name][value] = self._rows[row_name].pop(name) + self._cols[value] = self._cols.pop(name) + + def change_constraint_name(self, name, value): + super().change_constraint_name(name, value) + if name != value: + self._rows[value] = self._rows.pop(name) + for v in self._rows[value]: + self._cols[v].discard(name) + self._cols[v].add(value) + + def _update_variable_indices(self, start=0): + i = start + for name in islice(self._variables, start, None): + self._variables[name] = i + i += 1 + + +if _scipy_interface is not None: + + class _SparseModel(_scipy_interface.Model): + """``scipy_interface.Model`` on a `_SparseProblem` that copies and pickles sparsely. + + optlang pickles a model as JSON and restores it by adding the variables and + constraints to a fresh dense problem, which takes tens of minutes for a genome-scale + model. This class restores into a `_SparseProblem`, so ``copy.deepcopy``, + ``cobra.Model.copy`` and the worker processes of a pool get their copy in seconds. + """ + + # optlang finds its interface module (the scipy one) through this attribute. + __module__ = _scipy_interface.__name__ + + def _initialize_problem(self): + self.problem = _SparseProblem() + + def _remove_constraints(self, constraints): + # optlang's scipy interface calls the generic removal inside its loop, once per + # constraint with the whole list, so removing two or more constraints at once + # (up to 350) raises ``LookupError: Constraint ... not in solver``. + for constraint in constraints: + if constraint.lb is not None: + self.problem.remove_constraint(constraint.lower_constraint_name) + if constraint.ub is not None: + self.problem.remove_constraint(constraint.upper_constraint_name) + _scipy_interface.interface.Model._remove_constraints(self, constraints) + + def _initialize_configuration(self): + super()._initialize_configuration() + _install_tolerances(self) + + def __reduce__(self): + problem = self.problem + return (_restore_sparse_model, (self.to_json(), problem._feasibility, problem._optimality)) + + def _restore_sparse_model(state, feasibility, optimality): + model = _SparseModel() + model._init_from_json(state) + model.problem._feasibility = feasibility + model.problem._optimality = optimality + return model + + def _install_tolerances(solver): + """Let ``solver.configuration.tolerances`` (and so ``cobra.Model.tolerance``) set the + feasibility and optimality tolerance of HiGHS, which optlang's scipy interface ignores.""" + + def setter(attribute): + return lambda value: setattr(solver.problem, attribute, float(value)) + + solver.configuration.tolerances = SolverTolerances( + { + "feasibility": (lambda: solver.problem._feasibility or 1e-7, setter("_feasibility")), + "optimality": (lambda: solver.problem._optimality or 1e-7, setter("_optimality")), + } + ) + + +def is_scipy_interface(interface) -> bool: + """True if ``interface`` (a solver name, an optlang interface module or a model's + ``problem`` attribute) is optlang's scipy interface.""" + if _scipy_interface is None: + return False + if isinstance(interface, str): + return interface.lower() == "scipy" + return interface is _scipy_interface + + +def _is_scipy_problem(problem) -> bool: + return _scipy_interface is not None and isinstance(problem, _scipy_interface.Problem) + + +def _map_is_stale(problem) -> bool: + return list(problem._variables.values()) != list(range(len(problem._variables))) + + +def _adopt_dense_matrix(problem) -> None: + """Fill the dictionaries of a `_SparseProblem` from the dense matrix it was swapped from.""" + dense = problem._A + names = list(problem._variables) + row_names = list(problem._constraints) + problem._init_sparse() + problem._rows = {name: {} for name in row_names} + for r, c in zip(*np.nonzero(dense)): + problem._rows[row_names[r]][names[c]] = float(dense[r, c]) + problem._cols[names[c]].add(row_names[r]) + + +def _rebuild_from_expressions(problem, solver) -> None: + """Fill the dictionaries of a `_SparseProblem` from the constraint expressions of ``solver``. + + The expressions are the source the matrix is derived from, so this is correct whatever + happened to the matrix itself. + """ + problem._init_sparse() + problem._rows = {name: {} for name in problem._constraints} + for i, name in enumerate(problem._variables): + problem._variables[name] = i + for constraint in solver.constraints: + coefficients = None + for row_name, sign in ( + (constraint.upper_constraint_name, 1.0), + (constraint.lower_constraint_name, -1.0), + ): + if row_name not in problem._rows: + continue + if coefficients is None: + coefficients = constraint.coefficient_dict() + row = {v: sign * c for v, c in coefficients.items() if c != 0 and v in problem._variables} + problem._rows[row_name] = row + for v in row: + problem._cols[v].add(row_name) + + +def repair_scipy_problem(model) -> bool: + """Make the scipy problem of ``model`` correct and fast to edit. + + Does nothing and returns False when the solver of ``model`` is not optlang's scipy + interface. Otherwise it moves the problem to a class that renumbers the variable columns + correctly after a removal and stores the coefficients sparsely (see the module docstring), + applies the pending changes, restores the column map and the coefficient matrix if a + removal had already left the map stale, and returns True. Once the problem is switched, a + call is a type check. + + Parameters + ---------- + model + A ``cobra.Model``. A model class can call this + function already; call it yourself for a plain ``cobra.Model`` before you remove + variables (``remove_reactions``, ``Reaction.delete``, ``remove_cons_vars``). + """ + solver = getattr(model, "_solver", None) + problem = getattr(solver, "problem", None) + if not _is_scipy_problem(problem): + return False + if isinstance(problem, _SparseProblem): + return True + stale = _map_is_stale(problem) + problem.__class__ = _SparseProblem + solver.__class__ = _SparseModel + _install_tolerances(solver) + if stale: + problem._init_sparse() + problem._rows = {name: {} for name in problem._constraints} + else: + _adopt_dense_matrix(problem) + solver.update() + if stale: + _rebuild_from_expressions(problem, solver) + return True + + +def switch_to_scipy(model) -> None: + """Move ``model`` to the scipy solver, fast, with the sparse problem of this module. + + ``model.solver = "scipy"`` clones the problem through optlang, which grows a dense matrix + row by row. This builds the problem the way ``cobra.Model`` does when it is created, on + the sparse class, so a genome-scale model switches in seconds. The objective and its + direction are kept. A model whose solver holds anything besides one forward and one + reverse variable per reaction and one constraint per metabolite (an extra constraint or + variable of your own) is switched with optlang's clone instead and then repaired. + """ + if repair_scipy_problem(model): + return + solver = model.solver + plain = len(solver.constraints) == len(model.metabolites) and len(solver.variables) == 2 * len( + model.reactions + ) + if _scipy_interface is None or not plain: + from cobra import Model + + Model.solver.fset(model, "scipy") + repair_scipy_problem(model) + return + from cobra.util.solver import linear_reaction_coefficients + from optlang.symbolics import Zero + + coefficients = linear_reaction_coefficients(model) + direction = model.objective.direction + new = _SparseModel() + new.objective = _scipy_interface.Objective(Zero) + model._solver = new + _populate_sparse(model, new) + model.objective = coefficients + model.objective.direction = direction + if getattr(model, "_tolerance", None) is not None: + try: + model.tolerance = model._tolerance + except Exception: # the scipy interface has no tolerance settings to receive it + pass + + +def _populate_sparse(model, solver) -> None: + """Give ``solver`` one forward and one reverse variable per reaction of ``model`` and one + constraint per metabolite, as ``cobra.Model._populate_solver`` does. + + ``_populate_solver`` sets the coefficients of a constraint one at a time, and the scipy + interface removes and adds the rows of a constraint for each call, which takes hours for a + genome-scale model. Here every constraint is created with its whole expression and added + once. + """ + from optlang.symbolics import Zero, add + + problem = solver.interface + variables = {} + for reaction in model.reactions: + variables[reaction.id] = (problem.Variable(reaction.id), problem.Variable(reaction.reverse_id)) + solver.add([v for pair in variables.values() for v in pair]) + terms = {metabolite.id: [] for metabolite in model.metabolites} + for reaction in model.reactions: + forward, reverse = variables[reaction.id] + for metabolite, coefficient in reaction.metabolites.items(): + terms[metabolite.id].append(coefficient * forward) + terms[metabolite.id].append(-coefficient * reverse) + solver.add( + [ + problem.Constraint(add(*expressions) if expressions else Zero, name=name, lb=0, ub=0) + for name, expressions in terms.items() + ] + ) + for reaction in model.reactions: + reaction.update_variable_bounds() + + +@contextlib.contextmanager +def edit_on_fast_solver(model): + """Run structural edits of ``model`` on glpk when it is on the scipy solver. + + Removing reactions and metabolites, or rewriting the coefficients of thousands of + reactions, is quick on glpk and slow on the scipy interface, where optlang rewrites the + constraint expressions (and the rows) for each edit. Inside the block a scipy model is on + glpk; on exit it is converted back with :func:`switch_to_scipy`. The block changes nothing + for a model on another solver, for a model inside a ``with model:`` context (its + rollbacks hold the solver objects) and when glpk is unavailable. The optlang solver of + the model is a different object inside the block and after it. + """ + from cobra import Model + + solver = getattr(model, "_solver", None) + if not _is_scipy_problem(getattr(solver, "problem", None)) or getattr(model, "_contexts", None): + yield False + return + try: + Model.solver.fset(model, "glpk") + except Exception: + yield False + return + try: + yield True + finally: + switch_to_scipy(model) + + +@contextlib.contextmanager +def fast_solver_while_building(): + """Make models created inside the block on a solver that builds large models quickly. + + When cobra's default solver is scipy the block runs with glpk as the default, which + builds a genome-scale model in seconds; the default is restored on exit. Models created + inside are on glpk; convert them with :func:`switch_to_scipy`. With any other default + solver, or when glpk is unavailable, the block changes nothing. + """ + import cobra + + config = cobra.Configuration() + previous = config.solver + switched = False + if getattr(previous, "__name__", str(previous)).endswith("scipy_interface"): + try: + config.solver = "glpk" + switched = True + except Exception: + switched = False + try: + yield switched + finally: + if switched: + config.solver = previous + + +def _objective_vector(problem) -> np.ndarray: + """Cost vector ``linprog`` minimises: the objective, negated for a maximisation.""" + cost = np.array([problem.objective.get(name, 0.0) for name in problem._variables], dtype=float) + if problem.direction == "max": + cost = -cost + return cost + + +def _certified_result(problem): + """The stored ``linprog`` result if it is an optimum of ``problem`` as it is now, else None. + + Checks primal feasibility against the current rows and bounds, stationarity of the cost + vector against the stored marginals, and equality of the primal and dual objective. A + result that passes is an optimal primal-dual pair of the current problem, whatever solve + produced it. + """ + result = getattr(problem, "_solution", None) + try: + x = np.asarray(result.x, dtype=float) + y = np.asarray(result.ineqlin.marginals, dtype=float) + z_low = np.asarray(result.lower.marginals, dtype=float) + z_up = np.asarray(result.upper.marginals, dtype=float) + fun = float(result.fun) + matrix = sparse.csr_matrix(problem.A, dtype=float) + rhs = np.asarray(problem.upper_bounds, dtype=float) + bounds = list(problem.bounds.values()) + except (AttributeError, TypeError, ValueError): + return None + n, m = len(problem._variables), len(problem._constraints) + if matrix.shape != (m, n) or not (x.shape == z_low.shape == z_up.shape == (n,)) or y.shape != (m,): + return None + if len(bounds) != n or not np.all(np.isfinite(x)): + return None + low = np.array([-np.inf if b[0] is None else b[0] for b in bounds], dtype=float) + up = np.array([np.inf if b[1] is None else b[1] for b in bounds], dtype=float) + cost = _objective_vector(problem) + tol = _CERTIFICATE_TOL + + def close(a, b, scale): + return abs(a - b) <= tol * (1.0 + scale) + + if m and np.any(matrix @ x - rhs > tol * (1.0 + np.abs(rhs))): + return None + if np.any(x < low - tol * (1.0 + np.abs(low))) or np.any(x > up + tol * (1.0 + np.abs(up))): + return None + reduced = cost - (matrix.T @ y if m else 0.0) + if np.max(np.abs(reduced - (z_low + z_up)), initial=0.0) > tol * (1.0 + np.max(np.abs(cost), initial=0.0)): + return None + primal = float(cost @ x) + dual = float(rhs @ y) if m else 0.0 + dual += float(np.sum(low[np.isfinite(low)] * z_low[np.isfinite(low)])) + dual += float(np.sum(up[np.isfinite(up)] * z_up[np.isfinite(up)])) + if not (close(primal, fun, abs(fun)) and close(primal, dual, abs(primal))): + return None + return result + + +def constraint_shadow_prices(model, constraint_ids: Iterable[str]) -> np.ndarray: + """Shadow prices of the named constraints, on every solver, as one array. + + The shadow price of a constraint is the change of the optimum, in the direction of the + model objective, per unit rise of its right-hand side: for a metabolite balance + ``S v = 0`` it is the optimum gained per unit of the metabolite supplied. It is the value + of ``cobra.Metabolite.shadow_price`` for the glpk, gurobi and cplex interfaces, and for the + scipy interface (which raises) it is computed from the HiGHS row marginals of the last + solve. + + The model must have been solved. On the scipy interface the stored result is used only if + it is an optimum of the problem as it is now; otherwise the model is solved once more. If + that solve is not optimal, the function raises. + + Parameters + ---------- + model + A ``cobra.Model`` that has been optimized. + constraint_ids + IDs of constraints of the model, for example the metabolite ids of a pool constraint. + + Returns + ------- + numpy.ndarray + One shadow price per ID, in the order given. + + Raises + ------ + RuntimeError + If the scipy problem has no optimal solution to read the marginals from. + KeyError + If an ID is not a constraint of the model. + """ + ids = list(constraint_ids) + constraints = model.constraints + repair_scipy_problem(model) + problem = getattr(model.solver, "problem", None) + if _scipy_interface is None or not isinstance(problem, _scipy_interface.Problem): + return np.array([float(constraints[cid].dual) for cid in ids], dtype=float) + + result = _certified_result(problem) + if result is None: + model.slim_optimize() + result = _certified_result(problem) + if result is None: + raise RuntimeError( + "shadow prices need an optimal solution of the scipy problem; the model has none " + f"(solver status {model.solver.status!r})" + ) + marginals = np.asarray(result.ineqlin.marginals, dtype=float) + rows = problem._constraints + # The rows hold `A x <= ub` and `-A x <= -lb`. The marginal of a row is d(minimised value)/d(rhs + # of that row), so d/d(ub) is the upper row's marginal and d/d(lb) is minus the lower row's. + # A maximisation minimises the negated objective. + sign = -1.0 if problem.direction == "max" else 1.0 + prices = np.zeros(len(ids), dtype=float) + for k, cid in enumerate(ids): + constraint = constraints[cid] + value = 0.0 + if constraint.ub is not None: + value += marginals[rows[constraint.upper_constraint_name]] + if constraint.lb is not None: + value -= marginals[rows[constraint.lower_constraint_name]] + prices[k] = sign * value + return prices diff --git a/tests/test_annotation_smiles.py b/tests/test_annotation_smiles.py new file mode 100644 index 0000000..c9f780c --- /dev/null +++ b/tests/test_annotation_smiles.py @@ -0,0 +1,389 @@ +"""Tests for find_met_smiles (network mocked).""" +import logging + +import cobra + +from raven_toolbox.annotation import find_met_smiles + + +# --------------------------------------------------------------------------- # +# Fake HTTP layer +# --------------------------------------------------------------------------- # + +class _FakeResponse: + def __init__(self, status_code: int, text: str = ""): + self.status_code = status_code + self.text = text + + +def _install_fake_session(monkeypatch, name_to_response: dict): + """Replace _make_session so URL fetches return canned responses. + + `name_to_response` maps URL-escaped metabolite names to + (status_code, text). Names not in the map return 404. + """ + from raven_toolbox.annotation import smiles as mod + + class _FakeSession: + def get(self, url, timeout=None): + for fragment, (status, text) in name_to_response.items(): + if fragment in url: + return _FakeResponse(status, text) + return _FakeResponse(404, "") + + monkeypatch.setattr(mod, "_make_session", lambda: _FakeSession()) + # Skip the inter-request sleep in tests. + monkeypatch.setattr(mod, "_INTER_REQUEST_SLEEP", 0.0) + + +def _build_model(metabolites: list[tuple[str, str]]) -> cobra.Model: + """Build a cobra.Model from (id, name) tuples.""" + model = cobra.Model("test") + for met_id, met_name in metabolites: + m = cobra.Metabolite(met_id, name=met_name, compartment="c") + model.add_metabolites([m]) + return model + + +def _smiles(model: cobra.Model, met_id: str) -> str: + return model.metabolites.get_by_id(met_id).annotation.get("smiles", "") + + +# --------------------------------------------------------------------------- # +# Cache hits +# --------------------------------------------------------------------------- # + +def test_uses_cache_without_network(monkeypatch, tmp_path): + """If a SMILES is in the cache, no network call happens.""" + cache_path = tmp_path / "smilesDB.tsv" + cache_path.write_text("glucose\tOC1OC(CO)C(O)C(O)C1O\n") + + model = _build_model([("glc_c", "glucose")]) + + def _fail(): + raise AssertionError("network should not be called") + + from raven_toolbox.annotation import smiles as mod + monkeypatch.setattr(mod, "_make_session", _fail) + + find_met_smiles(model, cache_path=cache_path) + assert _smiles(model, "glc_c") == "OC1OC(CO)C(O)C(O)C1O" + + +def test_cache_with_empty_smiles_skips_network(monkeypatch, tmp_path): + """A cached empty result means 'we already know there is no SMILES'. + Should not retry the network for it.""" + cache_path = tmp_path / "smilesDB.tsv" + cache_path.write_text("unknown_compound\t\n") + + model = _build_model([("u_c", "unknown_compound")]) + + def _fail(): + raise AssertionError("network should not be called") + + from raven_toolbox.annotation import smiles as mod + monkeypatch.setattr(mod, "_make_session", _fail) + + find_met_smiles(model, cache_path=cache_path) + assert _smiles(model, "u_c") == "" + + +# --------------------------------------------------------------------------- # +# Network lookups +# --------------------------------------------------------------------------- # + +def test_fetches_from_network_when_cache_missing(monkeypatch, tmp_path): + cache_path = tmp_path / "smilesDB.tsv" + _install_fake_session(monkeypatch, { + "glucose": (200, "OC1OC(CO)C(O)C(O)C1O\n"), + }) + + model = _build_model([("glc_c", "glucose")]) + find_met_smiles(model, cache_path=cache_path) + + assert _smiles(model, "glc_c") == "OC1OC(CO)C(O)C(O)C1O" + + +def test_takes_first_line_when_multiple_returned(monkeypatch, tmp_path): + """PubChem may return multiple SMILES separated by newlines; the + function should keep only the first.""" + cache_path = tmp_path / "smilesDB.tsv" + _install_fake_session(monkeypatch, { + "ambiguous": (200, "C1CCCCC1\nCCCCCCC\nCCCC\n"), + }) + + model = _build_model([("a_c", "ambiguous")]) + find_met_smiles(model, cache_path=cache_path) + + assert _smiles(model, "a_c") == "C1CCCCC1" + + +def test_404_means_no_smiles(monkeypatch, tmp_path): + cache_path = tmp_path / "smilesDB.tsv" + _install_fake_session(monkeypatch, {}) # everything 404s + + model = _build_model([("x_c", "unknown_compound")]) + find_met_smiles(model, cache_path=cache_path) + + assert _smiles(model, "x_c") == "" + + +def test_500_treated_as_no_smiles(monkeypatch, tmp_path): + """A 500 (often returned for slashes etc. in names) is treated + as 'no match' rather than a crash.""" + cache_path = tmp_path / "smilesDB.tsv" + _install_fake_session(monkeypatch, { + "weird/name": (500, ""), + }) + model = _build_model([("w_c", "weird/name")]) + find_met_smiles(model, cache_path=cache_path) + assert _smiles(model, "w_c") == "" + + +# --------------------------------------------------------------------------- # +# Cache write-back +# --------------------------------------------------------------------------- # + +def test_appends_to_cache_after_fetch(monkeypatch, tmp_path): + cache_path = tmp_path / "smilesDB.tsv" + _install_fake_session(monkeypatch, { + "glucose": (200, "OC1OC(CO)C(O)C(O)C1O\n"), + }) + model = _build_model([("glc_c", "glucose")]) + find_met_smiles(model, cache_path=cache_path) + + text = cache_path.read_text() + assert "glucose\tOC1OC(CO)C(O)C(O)C1O\n" in text + + +def test_caches_empty_result_too(monkeypatch, tmp_path): + """Even when nothing is found, the empty result is cached so we + do not retry next time.""" + cache_path = tmp_path / "smilesDB.tsv" + _install_fake_session(monkeypatch, {}) + + model = _build_model([("x_c", "unknown")]) + find_met_smiles(model, cache_path=cache_path) + + text = cache_path.read_text() + assert "unknown\t\n" in text + + +def test_existing_cache_preserved_when_appending(monkeypatch, tmp_path): + """Existing cache entries should not be lost.""" + cache_path = tmp_path / "smilesDB.tsv" + cache_path.write_text("water\tO\n") + + _install_fake_session(monkeypatch, { + "ethanol": (200, "CCO\n"), + }) + model = _build_model([ + ("h2o_c", "water"), + ("etoh_c", "ethanol"), + ]) + find_met_smiles(model, cache_path=cache_path) + + text = cache_path.read_text() + assert "water\tO\n" in text + assert "ethanol\tCCO\n" in text + + +# --------------------------------------------------------------------------- # +# Filters and skip conditions +# --------------------------------------------------------------------------- # + +def test_skips_prot_metabolites(monkeypatch, tmp_path): + """Metabolite names starting with 'prot_' must not be looked up.""" + cache_path = tmp_path / "smilesDB.tsv" + + requested_urls: list[str] = [] + + class _CaptureSession: + def get(self, url, timeout=None): + requested_urls.append(url) + return _FakeResponse(404, "") + + from raven_toolbox.annotation import smiles as mod + monkeypatch.setattr(mod, "_make_session", lambda: _CaptureSession()) + monkeypatch.setattr(mod, "_INTER_REQUEST_SLEEP", 0.0) + + model = _build_model([ + ("prot_P1_c", "prot_P1"), + ("prot_pool_c", "prot_pool"), + ]) + find_met_smiles(model, cache_path=cache_path) + + assert requested_urls == [] + + +def test_skips_metabolites_with_existing_smiles(monkeypatch, tmp_path): + """If a metabolite already has annotation['smiles'], do not overwrite.""" + cache_path = tmp_path / "smilesDB.tsv" + _install_fake_session(monkeypatch, { + "glucose": (200, "should_not_be_used\n"), + }) + + model = _build_model([("glc_c", "glucose")]) + model.metabolites.get_by_id("glc_c").annotation["smiles"] = "PRESERVE_ME" + + find_met_smiles(model, cache_path=cache_path) + + assert _smiles(model, "glc_c") == "PRESERVE_ME" + + +def test_same_name_in_multiple_compartments_gets_same_smiles( + monkeypatch, tmp_path, +): + """Glucose in cytoplasm and extracellular compartments share a + name and therefore one PubChem lookup; both metabolites should + receive the SMILES.""" + cache_path = tmp_path / "smilesDB.tsv" + _install_fake_session(monkeypatch, { + "glucose": (200, "OC1OC(CO)C(O)C(O)C1O\n"), + }) + + model = cobra.Model("test") + m_c = cobra.Metabolite("glc_c", name="glucose", compartment="c") + m_e = cobra.Metabolite("glc_e", name="glucose", compartment="e") + model.add_metabolites([m_c, m_e]) + + find_met_smiles(model, cache_path=cache_path) + + assert _smiles(model, "glc_c") == "OC1OC(CO)C(O)C(O)C1O" + assert _smiles(model, "glc_e") == "OC1OC(CO)C(O)C(O)C1O" + + +def test_empty_model_does_not_crash(monkeypatch, tmp_path): + cache_path = tmp_path / "smilesDB.tsv" + model = cobra.Model("empty") + find_met_smiles(model, cache_path=cache_path) + + +# --------------------------------------------------------------------------- # +# Cache file creation +# --------------------------------------------------------------------------- # + +def test_creates_cache_file_if_missing(monkeypatch, tmp_path): + cache_path = tmp_path / "subdir" / "smilesDB.tsv" + _install_fake_session(monkeypatch, { + "glucose": (200, "C\n"), + }) + model = _build_model([("glc_c", "glucose")]) + find_met_smiles(model, cache_path=cache_path) + assert cache_path.is_file() + + +# --------------------------------------------------------------------------- # +# URL escaping +# --------------------------------------------------------------------------- # + +def test_url_escapes_special_characters(monkeypatch, tmp_path): + """A name with spaces or slashes should be URL-encoded before being + inserted into the URL.""" + cache_path = tmp_path / "smilesDB.tsv" + captured: list[str] = [] + + class _CaptureSession: + def get(self, url, timeout=None): + captured.append(url) + return _FakeResponse(404, "") + + from raven_toolbox.annotation import smiles as mod + monkeypatch.setattr(mod, "_make_session", lambda: _CaptureSession()) + monkeypatch.setattr(mod, "_INTER_REQUEST_SLEEP", 0.0) + + model = _build_model([("x", "L-alanyl-L-glycine")]) + find_met_smiles(model, cache_path=cache_path) + + assert len(captured) == 1 + # Hyphens are URL-safe; we just check the name is in the URL exactly + # (no spaces, no broken escaping). + assert "L-alanyl-L-glycine" in captured[0] + + +def test_url_escapes_spaces_and_slashes(monkeypatch, tmp_path): + cache_path = tmp_path / "smilesDB.tsv" + captured: list[str] = [] + + class _CaptureSession: + def get(self, url, timeout=None): + captured.append(url) + return _FakeResponse(404, "") + + from raven_toolbox.annotation import smiles as mod + monkeypatch.setattr(mod, "_make_session", lambda: _CaptureSession()) + monkeypatch.setattr(mod, "_INTER_REQUEST_SLEEP", 0.0) + + model = _build_model([("x", "weird name/here")]) + find_met_smiles(model, cache_path=cache_path) + + assert len(captured) == 1 + assert " " not in captured[0] + # Unescaped slash would still be valid in a URL but PubChem would + # interpret it as a path separator. Our quote(safe='') should + # escape it. + assert "weird%20name%2Fhere" in captured[0] + + +# --------------------------------------------------------------------------- # +# Logging +# --------------------------------------------------------------------------- # + +def test_logs_summary_after_run(monkeypatch, tmp_path, caplog): + cache_path = tmp_path / "smilesDB.tsv" + _install_fake_session(monkeypatch, { + "glucose": (200, "C\n"), + }) + model = _build_model([ + ("glc_c", "glucose"), + ("u_c", "unknown_compound"), + ]) + + with caplog.at_level(logging.INFO): + find_met_smiles(model, cache_path=cache_path) + + assert "SMILES found for" in caplog.text + + +# --------------------------------------------------------------------------- # +# Reading the annotation back +# --------------------------------------------------------------------------- # + +def test_met_smiles_accepts_the_shape_a_loaded_model_carries(): + """cobra returns a list for a model read from a file, a string for one + annotated in memory. Both mean the same molecule.""" + import cobra + + from raven_toolbox.annotation import met_smiles + + met = cobra.Metabolite("etoh") + met.annotation["smiles"] = "CCO" + assert met_smiles(met) == "CCO" + + met.annotation["smiles"] = ["CCO"] + assert met_smiles(met) == "CCO" + + met.annotation["smiles"] = [] + assert met_smiles(met) == "" + + del met.annotation["smiles"] + assert met_smiles(met) == "" + + +def test_duplicate_cache_name_uses_the_last_row(monkeypatch, tmp_path): + cache_path = tmp_path / "smilesDB.tsv" + cache_path.write_text("glucose\tFIRST\nglucose\tLAST\n") + model = _build_model([("glc_c", "glucose")]) + find_met_smiles(model, cache_path=cache_path) + assert _smiles(model, "glc_c") == "LAST" + + +def test_metabolite_without_name_is_skipped_with_a_warning(monkeypatch, tmp_path, caplog): + cache_path = tmp_path / "smilesDB.tsv" + cache_path.write_text("m1_c\tFROM_ID\nglucose\tCCO\n") + model = _build_model([("m1_c", ""), ("glc_c", "glucose")]) + with caplog.at_level(logging.WARNING): + find_met_smiles(model, cache_path=cache_path) + assert _smiles(model, "m1_c") == "" + assert _smiles(model, "glc_c") == "CCO" + assert any("m1_c" in r.getMessage() and "no name" in r.getMessage() for r in caplog.records) diff --git a/tests/test_biomass_maintenance.py b/tests/test_biomass_maintenance.py new file mode 100644 index 0000000..0266c77 --- /dev/null +++ b/tests/test_biomass_maintenance.py @@ -0,0 +1,310 @@ +"""Tests for the GAM and NGAM fit (``raven_toolbox.biomass.maintenance``). + +The toy model makes the uptake analytic. Glucose gives 2 ATP; the biomass reaction takes +5 glucose and ``GAM`` ATP per unit of biomass; the maintenance reaction takes ``NGAM`` ATP. +The glucose uptake at growth ``mu`` is therefore ``5 * mu + (GAM * mu + NGAM) / 2``. +""" +import cobra +import numpy as np +import pandas as pd +import pytest + +from raven_toolbox.biomass import ( + MaintenanceConfig, + bootstrap_gam, + fit_gam, + fit_maintenance, + get_gam, + huber_loss, + log_ratio, + min_uptake, + ngam_from_maintenance_coefficient, + set_gam, + study_weights, + uptake_grid, +) + +NAMES = ("ATP", "ADP", "H2O", "H+", "phosphate") +CONFIG = MaintenanceConfig( + biomass_rxn="BIO", ngam_rxn="NGAM", growth_rxn="GROWTH", + substrate_exchange="EX_glc", cofactor_met_names=NAMES, +) +PROTON_OFFSET = 0.3 # balances the charge of the other biomass terms + + +def _toy(gam=30.0, ngam=1.0): + model = cobra.Model("toy") + mets = { + i: cobra.Metabolite(i, name=n, compartment="c") + for i, n in [("glc", "glucose"), ("atp", "ATP"), ("adp", "ADP"), ("h2o", "H2O"), + ("h", "H+"), ("pi", "phosphate"), ("bio", "biomass")] + } + m = mets + + def rxn(rid, stoich, lb=0.0, ub=1000.0): + r = cobra.Reaction(rid) + r.add_metabolites({m[k]: v for k, v in stoich.items()}) + r.bounds = (lb, ub) + return r + + model.add_reactions([ + rxn("EX_glc", {"glc": -1}, -1000, 1000), + rxn("ATPPROD", {"glc": -1, "adp": -2, "pi": -2, "h": -2, "atp": 2, "h2o": 2}), + rxn("BIO", {"glc": -5, "atp": -gam, "h2o": -gam, "adp": gam, "pi": gam, + "h": gam + PROTON_OFFSET, "bio": 1}), + rxn("GROWTH", {"bio": -1}), + rxn("NGAM", {"atp": -1, "h2o": -1, "adp": 1, "pi": 1, "h": 1}, ngam, ngam), + rxn("EX_h2o", {"h2o": -1}, -1000, 1000), + rxn("EX_h", {"h": -1}, -1000, 1000), + rxn("EX_pi", {"pi": -1}, -1000, 1000), + ]) + model.objective = "GROWTH" + return model + + +def _uptake(mu, gam, ngam): + return 5 * mu + (gam * mu + ngam) / 2 + + +# --- set_gam and get_gam --------------------------------------------------------------- + + +def test_get_gam_is_the_atp_coefficient(): + assert get_gam(_toy(gam=30.0), biomass_rxn="BIO") == 30.0 + + +def test_get_gam_needs_one_atp_in_the_reaction(): + with pytest.raises(ValueError, match="needs exactly one"): + get_gam(_toy(), biomass_rxn="BIO", atp_met_name="nothing") + + +def test_keep_offsets_shifts_every_term_and_keeps_the_proton_offset(): + model = _toy(gam=30.0) + set_gam(model, 45.0, biomass_rxn="BIO", cofactor_met_names=NAMES, keep_offsets_of="ATP") + coef = {met.name: c for met, c in model.reactions.BIO.metabolites.items()} + assert coef["ATP"] == -45.0 and coef["H2O"] == -45.0 + assert coef["ADP"] == 45.0 and coef["phosphate"] == 45.0 + assert coef["H+"] == pytest.approx(45.0 + PROTON_OFFSET) + + +def test_without_keep_offsets_every_term_is_set_to_the_value(): + model = _toy(gam=30.0) + set_gam(model, 45.0, biomass_rxn="BIO", cofactor_met_names=NAMES) + coef = {met.name: c for met, c in model.reactions.BIO.metabolites.items()} + assert coef["H+"] == 45.0 + + +def test_keep_offsets_reverts_in_a_context(): + model = _toy(gam=30.0) + with model: + set_gam(model, 45.0, biomass_rxn="BIO", cofactor_met_names=NAMES, + ngam_rxn="NGAM", ngam_value=2.0, keep_offsets_of="ATP") + assert get_gam(model, biomass_rxn="BIO") == 45.0 + assert get_gam(model, biomass_rxn="BIO") == 30.0 + assert model.reactions.NGAM.bounds == (1.0, 1.0) + + +def test_keep_offsets_of_needs_one_metabolite_of_that_name_in_the_reaction(): + with pytest.raises(ValueError, match="needs exactly one"): + set_gam(_toy(), 45.0, biomass_rxn="BIO", cofactor_met_names=("nothing", "ATP"), + keep_offsets_of="nothing") + + +def test_get_gam_needs_the_biomass_reaction(): + with pytest.raises(KeyError): + get_gam(_toy(), biomass_rxn="NOPE") + + +def test_keep_offsets_sets_ngam_too(): + model = _toy(gam=30.0) + set_gam(model, 45.0, biomass_rxn="BIO", cofactor_met_names=NAMES, + ngam_rxn="NGAM", ngam_value=2.0, keep_offsets_of="ATP") + assert model.reactions.NGAM.bounds == (2.0, 2.0) + + +def test_keep_offsets_of_must_be_a_cofactor(): + with pytest.raises(ValueError, match="not one of the cofactor names"): + set_gam(_toy(), 45.0, biomass_rxn="BIO", cofactor_met_names=("ADP",), keep_offsets_of="ATP") + + +# --- loss, ratios, weights ---------------------------------------------------------------- + + +def test_huber_loss_is_quadratic_then_linear(): + r = np.array([0.0, 0.1, -0.1, 0.45]) + assert huber_loss(r, delta=0.15) == pytest.approx( + [0.0, 0.005, 0.005, 0.15 * (0.45 - 0.075)]) + + +def test_log_ratio_broadcasts_and_penalises_infeasible_rows(): + grid = np.array([[2.0, np.nan], [1.0, 4.0]]) + out = log_ratio(grid, np.array([1.0, 2.0]), infeasible=0.7) + assert out == pytest.approx(np.array([[np.log(2.0), 0.7], [0.0, np.log(2.0)]])) + + +def test_study_weights_count_studies_equally_and_discount(): + w = study_weights(["a"] * 4 + ["b"] * 4 + ["c"], + discounted=[False] * 4 + [True] * 4 + [False]) + assert w[:4] == pytest.approx(0.5) + assert w[4:8] == pytest.approx(0.25) + assert w[8] == pytest.approx(1.0) + + +def test_study_weights_without_discount(): + assert study_weights(["a", "a", "b"]) == pytest.approx([2 ** -0.5, 2 ** -0.5, 1.0]) + + +# --- simulation ------------------------------------------------------------------------------ + + +def test_min_uptake_matches_the_analytic_value_and_leaves_the_model_as_it_was(): + model = _toy(gam=30.0, ngam=1.0) + assert min_uptake(model, 0.1, CONFIG) == pytest.approx(_uptake(0.1, 30.0, 1.0)) + assert model.reactions.GROWTH.bounds == (0.0, 1000.0) + assert model.reactions.EX_glc.lower_bound == -1000.0 + assert model.objective.expression.has(model.reactions.GROWTH.forward_variable) + + +def test_min_uptake_is_nan_when_the_growth_rate_cannot_be_reached(): + model = _toy() + model.reactions.ATPPROD.upper_bound = 0.1 # too little ATP for the growth rate + assert np.isnan(min_uptake(model, 0.8, CONFIG)) + + +def test_min_uptake_runs_the_setup_inside_the_context(): + model = _toy(gam=30.0) + + def setup(m): + m.reactions.EX_glc.upper_bound = 0.0 + m.reactions.NGAM.bounds = (3.0, 3.0) + + assert min_uptake(model, 0.0, CONFIG, setup=setup) == pytest.approx(1.5) + assert model.reactions.NGAM.bounds == (1.0, 1.0) + + +def test_uptake_grid_covers_every_gam_and_row(): + model = _toy() + data = pd.DataFrame({"growth_rate": [0.05, 0.1, 0.2]}) + grid = uptake_grid(model, data, [20.0, 40.0], 0.8, CONFIG) + expected = [[_uptake(mu, g, 0.8) for mu in data["growth_rate"]] for g in (20.0, 40.0)] + assert grid == pytest.approx(np.array(expected)) + assert get_gam(model, biomass_rxn="BIO") == 30.0 + + +def test_uptake_grid_calls_the_condition_setup_for_each_row(): + model = _toy() + data = pd.DataFrame({"growth_rate": [0.1, 0.1], "extra": [0.0, 10.0]}) + + def setup(m, row): + m.reactions.EX_glc.upper_bound = row["extra"] # not binding: only checks the call + + seen = [] + + def spy(m, row): + seen.append(row["extra"]) + setup(m, row) + + uptake_grid(model, data, [30.0], 1.0, CONFIG, condition_setup=spy) + assert seen == [0.0, 10.0] + + +def test_ngam_from_maintenance_coefficient_inverts_the_uptake_at_zero_growth(): + model = _toy(gam=30.0, ngam=1.0) + ngam = ngam_from_maintenance_coefficient(model, 0.4, CONFIG) + assert ngam == pytest.approx(0.8) # uptake at zero growth is NGAM / 2 + assert model.reactions.NGAM.bounds == (1.0, 1.0) + + +def test_ngam_from_maintenance_coefficient_with_a_given_gam_keeps_offsets(): + model = _toy(gam=30.0) + assert ngam_from_maintenance_coefficient(model, 0.4, CONFIG, gam=50.0) == pytest.approx(0.8) + assert get_gam(model, biomass_rxn="BIO") == 30.0 + + +# --- fit ----------------------------------------------------------------------------------------- + +TRUE_GAM, TRUE_NGAM = 40.0, 0.8 + + +def _chemostats(noise=0.0, seed=1): + rng = np.random.default_rng(seed) + rows = [] + for study in "abcdef": + for mu in (0.05, 0.1, 0.2, 0.3): + up = _uptake(mu, TRUE_GAM, TRUE_NGAM) * (1 + noise * rng.standard_normal()) + rows.append({"study": study, "growth_rate": mu, "uptake": up}) + return pd.DataFrame(rows) + + +def test_fit_gam_recovers_the_gam_the_data_were_made_with(): + model = _toy(gam=30.0) + data = _chemostats() + gam, grid, ratios = fit_gam(model, data, TRUE_NGAM, CONFIG, gams=np.arange(30.0, 50.1, 2.5)) + assert gam == 40.0 + assert ratios.shape == (len(grid), len(data)) + assert ratios[list(grid).index(40.0)] == pytest.approx(0.0, abs=1e-9) + + +def test_fit_gam_one_outlier_study_moves_the_result_by_less_than_a_grid_step(): + """A study that reads three times the uptake of the others, with a quadratic loss, + would pull the best GAM far above 40; the Huber loss keeps it within one step.""" + model = _toy(gam=30.0) + data = _chemostats() + data.loc[data["study"] == "a", "uptake"] *= 3.0 + weights = study_weights(data["study"]) + gam, _, _ = fit_gam(model, data, TRUE_NGAM, CONFIG, gams=np.arange(30.0, 50.1, 2.5), + weights=weights) + assert abs(gam - 40.0) <= 2.5 + + +def test_bootstrap_gam_is_exact_without_noise_and_wider_with_a_dissenting_study(): + gams = np.arange(30.0, 50.1, 2.5) + model = _toy(gam=30.0) + data = _chemostats() + weights = study_weights(data["study"]) + _, grid, ratios = fit_gam(model, data, TRUE_NGAM, CONFIG, gams=gams, weights=weights) + assert bootstrap_gam(grid, ratios, data["study"], weights, n=50) == (40.0, 40.0) + + data.loc[data["study"] == "a", "uptake"] *= 1.5 + _, grid, ratios = fit_gam(model, data, TRUE_NGAM, CONFIG, gams=gams, weights=weights) + lo, hi = bootstrap_gam(grid, ratios, data["study"], weights, n=200) + assert lo <= 40.0 <= hi and hi > lo + + +def test_bootstrap_gam_is_reproducible_by_seed(): + gams = np.array([30.0, 40.0, 50.0]) + rng = np.random.default_rng(0) + ratios = rng.normal(size=(3, 12)) + study = list("aabbccddeeff") + w = np.ones(12) + assert bootstrap_gam(gams, ratios, study, w, seed=3) == bootstrap_gam(gams, ratios, study, w, seed=3) + + +def test_fit_maintenance_returns_the_fit_and_leaves_the_model_alone(): + model = _toy(gam=30.0) + data = _chemostats() + fit = fit_maintenance(model, data, TRUE_NGAM / 2, CONFIG, + gams=np.arange(30.0, 50.1, 2.5), bootstrap=30) + assert fit.ngam == pytest.approx(TRUE_NGAM) + assert fit.gam == 40.0 and fit.gam_p16 == 40.0 and fit.gam_p84 == 40.0 + assert fit.units == 24 and fit.studies == 6 + assert fit.median_error == pytest.approx(0.0, abs=1e-9) + assert get_gam(model, biomass_rxn="BIO") == 30.0 + assert model.reactions.NGAM.bounds == (1.0, 1.0) + + +def test_fit_maintenance_discounts_rows_and_sets_up_conditions(): + model = _toy(gam=30.0) + data = _chemostats() + data["note"] = np.where(data["study"] == "a", "figure", "table") + calls = [] + + def setup(m, row): + calls.append(row["study"]) + + fit = fit_maintenance( + model, data, TRUE_NGAM / 2, CONFIG, gams=np.arange(30.0, 50.1, 2.5), bootstrap=5, + discounted=data["note"] == "figure", condition_setup=setup, + ) + assert fit.gam == 40.0 + assert len(calls) == 24 * 9 # nine GAM values diff --git a/tests/test_experiments_conditions.py b/tests/test_experiments_conditions.py new file mode 100644 index 0000000..78210d5 --- /dev/null +++ b/tests/test_experiments_conditions.py @@ -0,0 +1,614 @@ +"""Tests for experiments.conditions and experiments.distance. + +A tiny two-carbon-source toy model with a known enzyme cap, so every +simulated growth/exchange-flux value used below is hand-predictable +from the model's stoichiometry rather than treated as a black box. +""" +import cobra +import numpy as np +import pytest + +from raven_toolbox.experiments.data import ExperimentData +from raven_toolbox.experiments.flux_data import FluxData +from raven_toolbox.experiments.distance import ( + BIOMASS_CARBON_EQUIV, + experiment_distance, + compute_carbon_weights, + dataset_rmse, +) +from raven_toolbox.experiments.conditions import ( + ConditionSimResult, + simulate_conditions, +) + + +# --------------------------------------------------------------------------- # +# Toy two-carbon-source model fixture +# --------------------------------------------------------------------------- # +# +# glucose and ethanol each feed a shared pool metabolite A_c, which the +# single enzyme-gated reaction R turns into biomass at an enzyme-capped +# rate of kcat*3600/mw = 1*3600/100 = 36 (pool size 1 mg/gDW). R also +# co-produces a small fixed amount of a "byproduct" the experimental +# data assumes is never made (byp_c, 0.1 mol per mol of R flux) -- this +# exercises `zero_flux_rxns`' nonzero-simulated-flux case. +# +# - "glucose" condition: uptake unconstrained relative to the enzyme +# cap (measured flux -36 exactly matches TR's capacity), so growth +# is enzyme-limited: 36. +# - "ethanol" condition: uptake capped at -20 (below the enzyme cap), +# so growth is carbon-limited instead: 20. + +_KCAT = 1.0 +_MW = 100.0 +_ENZYME_CAP = _KCAT * 3600.0 / _MW # 36.0 +_BYPRODUCT_YIELD = 0.1 + + +def _build_toy() -> cobra.Model: + model = cobra.Model("toy") + + glc_e = cobra.Metabolite("glc_e", compartment="e", formula="C6H12O6") + eth_e = cobra.Metabolite("eth_e", compartment="e", formula="C2H6O") + byp_c = cobra.Metabolite("byp_c", compartment="c", formula="C1") + A_c = cobra.Metabolite("A_c", compartment="c") + pool = cobra.Metabolite("prot_pool", compartment="c") + prot_E = cobra.Metabolite("prot_E", compartment="c") + bio_met = cobra.Metabolite("bio_met", compartment="c") + model.add_metabolites([glc_e, eth_e, byp_c, A_c, pool, prot_E, bio_met]) + + EX_glc = cobra.Reaction("EX_glc") + EX_glc.add_metabolites({glc_e: -1.0}) + EX_glc.bounds = (-1000.0, 0.0) + + EX_eth = cobra.Reaction("EX_eth") + EX_eth.add_metabolites({eth_e: -1.0}) + EX_eth.bounds = (-1000.0, 0.0) + + EX_byp = cobra.Reaction("EX_byp") + EX_byp.add_metabolites({byp_c: -1.0}) + EX_byp.bounds = (0.0, 1000.0) + + TR_glc = cobra.Reaction("TR_glc") + TR_glc.add_metabolites({glc_e: -1.0, A_c: 1.0}) + TR_glc.bounds = (0.0, 1000.0) + + TR_eth = cobra.Reaction("TR_eth") + TR_eth.add_metabolites({eth_e: -1.0, A_c: 1.0}) + TR_eth.bounds = (0.0, 1000.0) + + coeff = _MW / (_KCAT * 3600.0) # mg/mmol per unit flux + R = cobra.Reaction("R") + R.add_metabolites( + {A_c: -1.0, prot_E: -coeff, bio_met: 1.0, byp_c: _BYPRODUCT_YIELD} + ) + R.bounds = (0.0, 1000.0) + + BIO = cobra.Reaction("biomass") + BIO.add_metabolites({bio_met: -1.0}) + BIO.bounds = (0.0, 1000.0) + + pool_ex = cobra.Reaction("prot_pool_exchange") + pool_ex.add_metabolites({pool: 1.0}) + pool_ex.bounds = (0.0, 1.0) # 1 mg/gDW pool + + usage_E = cobra.Reaction("usage_prot_E") + usage_E.add_metabolites({pool: -1.0, prot_E: 1.0}) + usage_E.bounds = (0.0, 1000.0) + + model.add_reactions( + [EX_glc, EX_eth, EX_byp, TR_glc, TR_eth, R, BIO, pool_ex, usage_E] + ) + model.objective = "biomass" + return model + + +def _flux_data(*, ethanol_grrate: float) -> FluxData: + """Two conditions: glucose (enzyme-limited, everything matches + exactly) and ethanol (carbon-limited; ``ethanol_grrate`` controls + whether the measured growth matches the simulated 20.0 exactly or + deliberately deviates from it).""" + return FluxData( + conds=["glucose", "ethanol"], + p_tot=np.array([np.nan, np.nan]), + gr_rate=np.array([_ENZYME_CAP, ethanol_grrate]), + exch_fluxes=np.array( + [ + [-_ENZYME_CAP, np.nan], + [np.nan, -20.0], + ] + ), + exch_mets=["glucose", "ethanol"], + exch_rxn_ids=["EX_glc", "EX_eth"], + weight=np.array([1.0, 1.0]), + source=["test", "test"], + ) + + +def _max_growth_data() -> FluxData: + """Max-growth dataset: uptake fully opened, so both conditions hit + the same enzyme-capped growth of 36.""" + return FluxData( + conds=["glucose", "ethanol"], + p_tot=np.array([np.nan, np.nan]), + gr_rate=np.array([_ENZYME_CAP, _ENZYME_CAP]), + exch_fluxes=np.array( + [ + [-1000.0, np.nan], + [np.nan, -1000.0], + ] + ), + exch_mets=["glucose", "ethanol"], + exch_rxn_ids=["EX_glc", "EX_eth"], + ) + + +# --------------------------------------------------------------------------- # +# simulate_conditions +# --------------------------------------------------------------------------- # + +def test_simulate_flux_data_enzyme_vs_carbon_limited(): + model = _build_toy() + flux_data = _flux_data(ethanol_grrate=20.0) + + results = simulate_conditions( + model, flux_data, + constrain=True, zero_flux_rxns=["EX_byp"], bio_rxn_id="biomass", + ) + + assert len(results) == 2 + glucose, ethanol = results + + # Glucose: enzyme-limited growth of 36, full carbon uptake used. + assert glucose.feasible + assert glucose.growth == pytest.approx(_ENZYME_CAP) + assert glucose.exch_fluxes["EX_glc"] == pytest.approx(-_ENZYME_CAP) + assert glucose.block_fluxes["EX_byp"] == pytest.approx( + _ENZYME_CAP * _BYPRODUCT_YIELD + ) + + # Ethanol: carbon-limited to the -20 uptake cap, growth = 20. + assert ethanol.feasible + assert ethanol.growth == pytest.approx(20.0) + assert ethanol.exch_fluxes["EX_eth"] == pytest.approx(-20.0) + assert ethanol.block_fluxes["EX_byp"] == pytest.approx(20.0 * _BYPRODUCT_YIELD) + + # Bounds fully revert after the call (both `with model:` blocks exited). + assert model.reactions.get_by_id("EX_glc").bounds == (-1000.0, 0.0) + assert model.reactions.get_by_id("EX_eth").bounds == (-1000.0, 0.0) + + +def test_simulate_max_growth_opens_uptake_fully(): + model = _build_toy() + max_growth = _max_growth_data() + + results = simulate_conditions( + model, max_growth, + constrain=False, zero_flux_rxns=[], bio_rxn_id="biomass", + ) + + assert [r.growth for r in results] == pytest.approx([_ENZYME_CAP, _ENZYME_CAP]) + # max_growth rows carry no exchange-flux measurements to report back. + assert results[0].exch_fluxes == {} + assert results[0].block_fluxes == {} + + +def test_simulate_unmatched_condition_name_raises(): + model = _build_toy() + flux_data = _flux_data(ethanol_grrate=20.0) + flux_data.conds = ["glucose", "propionate"] + + with pytest.raises(ValueError, match="propionate"): + simulate_conditions( + model, flux_data, + constrain=True, zero_flux_rxns=[], bio_rxn_id="biomass", + ) + + +# --------------------------------------------------------------------------- # +# compute_carbon_weights +# --------------------------------------------------------------------------- # + +def test_compute_carbon_weights(): + model = _build_toy() + + carbon_weights = compute_carbon_weights( + model, ["EX_glc", "EX_eth", "EX_byp", "biomass"], bio_rxn_id="biomass", + ) + + assert carbon_weights["EX_glc"] == pytest.approx(6.0) # C6H12O6 + assert carbon_weights["EX_eth"] == pytest.approx(2.0) # C2H6O + assert carbon_weights["EX_byp"] == pytest.approx(1.0) # C1 + assert carbon_weights["biomass"] == pytest.approx(BIOMASS_CARBON_EQUIV) + + +def test_compute_carbon_weights_zero_carbon_clamps_to_one(): + """A genuinely zero-carbon exchanged metabolite (e.g. O2) still + gets weight 1, not 0, so those reactions' mismatches are not silently + excluded from the RMSE.""" + model = _build_toy() + o2_e = cobra.Metabolite("o2_e", compartment="e", formula="O2") + model.add_metabolites([o2_e]) + EX_o2 = cobra.Reaction("EX_o2") + EX_o2.add_metabolites({o2_e: -1.0}) + EX_o2.bounds = (-1000.0, 0.0) + model.add_reactions([EX_o2]) + + carbon_weights = compute_carbon_weights(model, ["EX_o2"], bio_rxn_id="biomass") + + assert carbon_weights["EX_o2"] == pytest.approx(1.0) + + +# --------------------------------------------------------------------------- # +# dataset_rmse / experiment_distance +# --------------------------------------------------------------------------- # + +def test_dataset_rmse_measured_terms_match_only_byproduct_deviates(): + model = _build_toy() + # ethanol_grrate=20.0 matches the ethanol condition's actual + # carbon-limited growth exactly, and both conditions' own measured + # exchange fluxes match the simulation exactly too -- the only + # nonzero term left is the unmeasured byproduct (assumed + # zero-flux, but nonzero in the simulation). + flux_data = _flux_data(ethanol_grrate=20.0) + sims = simulate_conditions( + model, flux_data, + constrain=True, zero_flux_rxns=["EX_byp"], bio_rxn_id="biomass", + ) + carbon_weights = compute_carbon_weights( + model, ["EX_glc", "EX_eth", "EX_byp", "biomass"], bio_rxn_id="biomass", + ) + + rmse, per_cond = dataset_rmse( + flux_data, sims, + constrain=True, carbon_weights=carbon_weights, bio_rxn_id="biomass", + ) + + # The byproduct term is nonzero-simulated-vs-zero-measured, so RMSE + # is not zero overall -- but every *measured* exchange/growth term + # matches exactly, and the byproduct term is hand-computable. + glucose_byp = _ENZYME_CAP * _BYPRODUCT_YIELD * 1.0 # carbon weight 1 + ethanol_byp = 20.0 * _BYPRODUCT_YIELD * 1.0 + # 3 terms per condition: growth (matches exactly -> 0), the + # condition's own exchange flux (matches exactly -> 0), and the + # byproduct block term (measured 0 vs the nonzero simulated value). + expected_glucose_rmse = np.sqrt(np.mean([0.0, 0.0, glucose_byp ** 2])) + expected_ethanol_rmse = np.sqrt(np.mean([0.0, 0.0, ethanol_byp ** 2])) + + assert per_cond[0] == pytest.approx(expected_glucose_rmse) + assert per_cond[1] == pytest.approx(expected_ethanol_rmse) + assert rmse == pytest.approx(np.mean([expected_glucose_rmse, expected_ethanol_rmse])) + + +def test_dataset_rmse_growth_mismatch_is_hand_computable(): + model = _build_toy() + # Deliberately wrong measured growth for the ethanol condition + # (18.0 vs the actual carbon-limited 20.0) -- a known, hand-provable + # deviation on top of the byproduct term. + flux_data = _flux_data(ethanol_grrate=18.0) + sims = simulate_conditions( + model, flux_data, + constrain=True, zero_flux_rxns=["EX_byp"], bio_rxn_id="biomass", + ) + carbon_weights = compute_carbon_weights( + model, ["EX_glc", "EX_eth", "EX_byp", "biomass"], bio_rxn_id="biomass", + ) + + _, per_cond = dataset_rmse( + flux_data, sims, + constrain=True, carbon_weights=carbon_weights, bio_rxn_id="biomass", + ) + + growth_diff = (18.0 - 20.0) * BIOMASS_CARBON_EQUIV # bio_meas - bio_sim + byp_diff = 0.0 - (20.0 * _BYPRODUCT_YIELD) # measured 0 vs simulated + expected = np.sqrt(np.mean([growth_diff ** 2, 0.0, byp_diff ** 2])) + assert per_cond[1] == pytest.approx(expected) + + +def test_dataset_rmse_infeasible_condition_gets_penalty(): + flux_data = _flux_data(ethanol_grrate=20.0) + sims = [ConditionSimResult(feasible=False), ConditionSimResult(feasible=False)] + carbon_weights = {} + + rmse, per_cond = dataset_rmse( + flux_data, sims, + constrain=True, carbon_weights=carbon_weights, bio_rxn_id="biomass", penalty=99.0, + ) + + assert list(per_cond) == [99.0, 99.0] + assert rmse == pytest.approx(99.0) + + +def test_dataset_rmse_max_growth_scores_growth_only(): + max_growth = _max_growth_data() + sims = [ + ConditionSimResult(feasible=True, growth=_ENZYME_CAP), + ConditionSimResult(feasible=True, growth=30.0), # deliberately off + ] + + rmse, per_cond = dataset_rmse( + max_growth, sims, + constrain=False, carbon_weights={}, bio_rxn_id="biomass", + ) + + assert per_cond[0] == pytest.approx(0.0) + expected_second = abs((_ENZYME_CAP - 30.0) * BIOMASS_CARBON_EQUIV) + assert per_cond[1] == pytest.approx(expected_second) + assert rmse == pytest.approx(np.mean([0.0, expected_second])) + + +def test_experiment_distance_combines_both_datasets(): + model = _build_toy() + flux_data = _flux_data(ethanol_grrate=20.0) + max_growth = _max_growth_data() + experiment_data = ExperimentData(flux_data=flux_data, max_growth=max_growth, zero_flux=["EX_byp"]) + + flux_sims = simulate_conditions( + model, flux_data, + constrain=True, zero_flux_rxns=["EX_byp"], bio_rxn_id="biomass", + ) + max_growth_sims = simulate_conditions( + model, max_growth, + constrain=False, zero_flux_rxns=[], bio_rxn_id="biomass", + ) + carbon_weights = compute_carbon_weights( + model, ["EX_glc", "EX_eth", "EX_byp", "biomass"], bio_rxn_id="biomass", + ) + + rmse, detail = experiment_distance( + experiment_data, + flux_sims=flux_sims, max_growth_sims=max_growth_sims, + carbon_weights=carbon_weights, bio_rxn_id="biomass", + ) + + expected_flux_rmse, _ = dataset_rmse( + flux_data, flux_sims, constrain=True, carbon_weights=carbon_weights, bio_rxn_id="biomass", + ) + expected_max_growth_rmse, _ = dataset_rmse( + max_growth, max_growth_sims, constrain=False, carbon_weights=carbon_weights, + bio_rxn_id="biomass", + ) + assert rmse == pytest.approx( + np.mean([expected_flux_rmse, expected_max_growth_rmse]) + ) + assert "flux_data" in detail and "max_growth" in detail + + +def test_experiment_distance_max_growth_weight_scales_the_max_growth_term(): + """``max_growth_weight`` scales the max-growth term, so 2 makes the + eight max-growth conditions count double against the 33 flux ones.""" + model = _build_toy() + flux_data = _flux_data(ethanol_grrate=20.0) + max_growth = _max_growth_data() + experiment_data = ExperimentData(flux_data=flux_data, max_growth=max_growth, zero_flux=["EX_byp"]) + + flux_sims = simulate_conditions( + model, flux_data, + constrain=True, zero_flux_rxns=["EX_byp"], bio_rxn_id="biomass", + ) + max_growth_sims = simulate_conditions( + model, max_growth, + constrain=False, zero_flux_rxns=[], bio_rxn_id="biomass", + ) + carbon_weights = compute_carbon_weights( + model, ["EX_glc", "EX_eth", "EX_byp", "biomass"], bio_rxn_id="biomass", + ) + + expected_flux_rmse, _ = dataset_rmse( + flux_data, flux_sims, constrain=True, carbon_weights=carbon_weights, bio_rxn_id="biomass", + ) + expected_max_growth_rmse, _ = dataset_rmse( + max_growth, max_growth_sims, constrain=False, carbon_weights=carbon_weights, + bio_rxn_id="biomass", + ) + + rmse, _ = experiment_distance( + experiment_data, + flux_sims=flux_sims, max_growth_sims=max_growth_sims, + carbon_weights=carbon_weights, bio_rxn_id="biomass", max_growth_weight=2.0, + ) + assert rmse == pytest.approx( + (expected_flux_rmse + 2.0 * expected_max_growth_rmse) / 3.0 + ) + + # 0.5 weights the flux term twice as much as the max-growth term. + rmse_half, _ = experiment_distance( + experiment_data, + flux_sims=flux_sims, max_growth_sims=max_growth_sims, + carbon_weights=carbon_weights, bio_rxn_id="biomass", max_growth_weight=0.5, + ) + assert rmse_half == pytest.approx( + (2.0 * expected_flux_rmse + expected_max_growth_rmse) / 3.0 + ) + + # A lone dataset carries the whole score whatever the weight. + solo = ExperimentData(flux_data=flux_data, max_growth=None, zero_flux=["EX_byp"]) + rmse_solo, _ = experiment_distance( + solo, + flux_sims=flux_sims, max_growth_sims=None, + carbon_weights=carbon_weights, bio_rxn_id="biomass", max_growth_weight=2.0, + ) + assert rmse_solo == pytest.approx(expected_flux_rmse) + + +def test_experiment_distance_missing_dataset_contributes_nothing(): + model = _build_toy() + flux_data = _flux_data(ethanol_grrate=20.0) + experiment_data = ExperimentData(flux_data=flux_data, max_growth=None, zero_flux=["EX_byp"]) + + flux_sims = simulate_conditions( + model, flux_data, + constrain=True, zero_flux_rxns=["EX_byp"], bio_rxn_id="biomass", + ) + carbon_weights = compute_carbon_weights( + model, ["EX_glc", "EX_eth", "EX_byp", "biomass"], bio_rxn_id="biomass", + ) + + rmse, detail = experiment_distance( + experiment_data, + flux_sims=flux_sims, max_growth_sims=None, + carbon_weights=carbon_weights, bio_rxn_id="biomass", + ) + + expected_flux_rmse, _ = dataset_rmse( + flux_data, flux_sims, constrain=True, carbon_weights=carbon_weights, bio_rxn_id="biomass", + ) + assert rmse == pytest.approx(expected_flux_rmse) + assert set(detail) == {"flux_data"} + + +def test_simulate_uses_the_given_optimize_function(): + """``optimize`` replaces ``model.optimize()`` for every condition.""" + model = _build_toy() + calls = [] + + def optimize(m): + calls.append(m) + return m.optimize() + + sims = simulate_conditions( + model, _flux_data(ethanol_grrate=20.0), + constrain=True, zero_flux_rxns=[], bio_rxn_id="biomass", + optimize=optimize, + ) + + assert len(calls) == 2 + assert all(s.feasible for s in sims) + + +# --------------------------------------------------------------------------- # +# Conditions with weight 0 are left out +# --------------------------------------------------------------------------- # + +def _with_std_row(flux_data: FluxData) -> FluxData: + """The data with a first condition named ``Std`` that matches no exchange and has + weight 0, as the condition of a protein measurement in a tutorial file.""" + n_rxns = len(flux_data.exch_rxn_ids) + weight = np.ones(len(flux_data.conds)) if flux_data.weight is None else flux_data.weight + return FluxData( + conds=["Std"] + list(flux_data.conds), + p_tot=np.concatenate([[0.5], flux_data.p_tot]), + gr_rate=np.concatenate([[0.1], flux_data.gr_rate]), + exch_fluxes=np.vstack([np.full((1, n_rxns), np.nan), flux_data.exch_fluxes]), + exch_mets=list(flux_data.exch_mets), + exch_rxn_ids=list(flux_data.exch_rxn_ids), + weight=np.concatenate([[0.0], weight]), + ) + + +def _simulate(model, flux_data, *, constrain): + return simulate_conditions( + model, flux_data, + constrain=constrain, zero_flux_rxns=["EX_byp"] if constrain else [], + bio_rxn_id="biomass", + ) + + +def test_a_condition_with_weight_zero_is_not_simulated_or_matched(): + model = _build_toy() + flux_data = _flux_data(ethanol_grrate=20.0) + with_std = _with_std_row(flux_data) + + sims = _simulate(model, with_std, constrain=True) + reference = _simulate(_build_toy(), flux_data, constrain=True) + + assert len(sims) == 3 + assert sims[0].skipped and not sims[0].feasible + assert [s.skipped for s in sims[1:]] == [False, False] + assert [s.growth for s in sims[1:]] == [s.growth for s in reference] + + +def test_a_condition_with_weight_zero_does_not_block_its_exchange(): + model = _build_toy() + flux_data = _flux_data(ethanol_grrate=20.0) + seen = [] + + def optimize(m): + seen.append(m.reactions.get_by_id("EX_eth").lower_bound) + return m.optimize() + + flux_data.weight = np.array([1.0, 0.0]) + _simulate_with(model, flux_data, optimize) + assert seen == [-1000.0] + + seen.clear() + flux_data.weight = np.array([1.0, 1.0]) + _simulate_with(model, flux_data, optimize) + assert seen == [0.0, -20.0] + + +def _simulate_with(model, flux_data, optimize): + return simulate_conditions( + model, flux_data, constrain=True, zero_flux_rxns=[], bio_rxn_id="biomass", + optimize=optimize, + ) + + +@pytest.mark.parametrize("constrain", [True, False]) +def test_a_condition_with_weight_zero_does_not_change_the_distance(constrain): + model = _build_toy() + flux_data = _flux_data(ethanol_grrate=20.0) if constrain else _max_growth_data() + with_std = _with_std_row(flux_data) + carbon_weights = compute_carbon_weights( + model, ["EX_glc", "EX_eth", "EX_byp", "biomass"], bio_rxn_id="biomass", + ) + + rmse, per_cond = dataset_rmse( + with_std, _simulate(model, with_std, constrain=constrain), + constrain=constrain, carbon_weights=carbon_weights, bio_rxn_id="biomass", + ) + expected, expected_per_cond = dataset_rmse( + flux_data, _simulate(_build_toy(), flux_data, constrain=constrain), + constrain=constrain, carbon_weights=carbon_weights, bio_rxn_id="biomass", + ) + + assert rmse == expected + assert np.isnan(per_cond[0]) + assert list(per_cond[1:]) == list(expected_per_cond) + + +def test_experiment_distance_leaves_out_a_dataset_without_active_conditions(): + model = _build_toy() + flux_data = _flux_data(ethanol_grrate=20.0) + flux_data.weight = np.array([0.0, 0.0]) + max_growth = _max_growth_data() + experiment_data = ExperimentData(flux_data=flux_data, max_growth=max_growth) + carbon_weights = compute_carbon_weights( + model, ["EX_glc", "EX_eth", "biomass"], bio_rxn_id="biomass", + ) + flux_sims = _simulate(model, flux_data, constrain=True) + max_growth_sims = _simulate(model, max_growth, constrain=False) + assert all(s.skipped for s in flux_sims) + + rmse, detail = experiment_distance( + experiment_data, flux_sims=flux_sims, max_growth_sims=max_growth_sims, + carbon_weights=carbon_weights, bio_rxn_id="biomass", + ) + expected, _ = dataset_rmse( + max_growth, max_growth_sims, constrain=False, carbon_weights=carbon_weights, + bio_rxn_id="biomass", + ) + + assert rmse == expected + assert np.isnan(detail["flux_data"]).all() + + +def test_the_distance_is_nan_when_every_condition_has_weight_zero(): + model = _build_toy() + flux_data = _flux_data(ethanol_grrate=20.0) + flux_data.weight = np.array([0.0, 0.0]) + carbon_weights = compute_carbon_weights( + model, ["EX_glc", "EX_eth", "biomass"], bio_rxn_id="biomass", + ) + sims = _simulate(model, flux_data, constrain=True) + + rmse, per_cond = dataset_rmse( + flux_data, sims, constrain=True, carbon_weights=carbon_weights, bio_rxn_id="biomass", + ) + distance, _ = experiment_distance( + ExperimentData(flux_data=flux_data), flux_sims=sims, max_growth_sims=None, + carbon_weights=carbon_weights, bio_rxn_id="biomass", + ) + + assert np.isnan(rmse) and np.isnan(per_cond).all() + assert np.isnan(distance) diff --git a/tests/test_experiments_constrain.py b/tests/test_experiments_constrain.py new file mode 100644 index 0000000..72d5541 --- /dev/null +++ b/tests/test_experiments_constrain.py @@ -0,0 +1,329 @@ +"""Tests for apply_flux_data_constraints.""" +import cobra +import numpy as np +import pytest + +from raven_toolbox.experiments import FluxData, apply_flux_data_constraints + + +# --------------------------------------------------------------------------- # +# Fixture builders +# --------------------------------------------------------------------------- # + +def _build_simple_model() -> cobra.Model: + """Tiny model with biomass + a few exchange rxns.""" + model = cobra.Model("toy") + + glc = cobra.Metabolite("glc_e", compartment="e") + O2 = cobra.Metabolite("O2_e", compartment="e") + eth = cobra.Metabolite("eth_e", compartment="e") + bio_met = cobra.Metabolite("bio", compartment="c") + model.add_metabolites([glc, O2, eth, bio_met]) + + EX_glc = cobra.Reaction("EX_glc") + EX_glc.add_metabolites({glc: -1.0}) + EX_glc.lower_bound = -1000.0 + EX_glc.upper_bound = 1000.0 + + EX_O2 = cobra.Reaction("EX_O2") + EX_O2.add_metabolites({O2: -1.0}) + EX_O2.lower_bound = -1000.0 + EX_O2.upper_bound = 1000.0 + + EX_eth = cobra.Reaction("EX_eth") + EX_eth.add_metabolites({eth: -1.0}) + EX_eth.lower_bound = -1000.0 + EX_eth.upper_bound = 1000.0 + + BIO = cobra.Reaction("biomass") + BIO.add_metabolites({bio_met: 1.0}) + BIO.lower_bound = 0.0 + BIO.upper_bound = 1000.0 + + model.add_reactions([EX_glc, EX_O2, EX_eth, BIO]) + + return model + + +def _flux_data( + *, + conds: list[str], + p_tot: list[float], + gr_rate: list[float], + exch_fluxes: list[list[float]], + exch_rxn_ids: list[str], +) -> FluxData: + return FluxData( + conds=list(conds), + p_tot=np.array(p_tot, dtype=float), + gr_rate=np.array(gr_rate, dtype=float), + exch_fluxes=np.array(exch_fluxes, dtype=float), + exch_mets=list(exch_rxn_ids), + exch_rxn_ids=list(exch_rxn_ids), + ) + + +# --------------------------------------------------------------------------- # +# Pre-flight +# --------------------------------------------------------------------------- # + +def test_invalid_max_min_growth_raises(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-5.0]], exch_rxn_ids=["EX_glc"], + ) + with pytest.raises(ValueError, match="max_min_growth"): + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", max_min_growth="invalid") + + +def test_invalid_loose_strict_raises(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-5.0]], exch_rxn_ids=["EX_glc"], + ) + with pytest.raises(ValueError, match="loose_strict_flux"): + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", loose_strict_flux=200) + + +def test_unknown_condition_name_raises(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-5.0]], exch_rxn_ids=["EX_glc"], + ) + with pytest.raises(ValueError, match="Condition"): + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", condition="nonexistent") + + +def test_out_of_range_condition_index_raises(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-5.0]], exch_rxn_ids=["EX_glc"], + ) + with pytest.raises(IndexError, match="condition index"): + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", condition=5) + + +def test_unknown_rxn_id_raises(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-5.0]], exch_rxn_ids=["EX_nonexistent"], + ) + with pytest.raises(ValueError, match="not present in the model"): + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass") + + +# --------------------------------------------------------------------------- # +# Growth bounds +# --------------------------------------------------------------------------- # + +def test_max_growth_sets_upper_bound(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-5.0]], exch_rxn_ids=["EX_glc"], + ) + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", max_min_growth="max") + bio = model.reactions.get_by_id("biomass") + assert bio.lower_bound == 0.0 + assert bio.upper_bound == pytest.approx(0.4) + + +def test_min_growth_sets_lower_bound(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-5.0]], exch_rxn_ids=["EX_glc"], + ) + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", max_min_growth="min") + bio = model.reactions.get_by_id("biomass") + assert bio.lower_bound == pytest.approx(0.4) + assert bio.upper_bound == 1000.0 + + +# --------------------------------------------------------------------------- # +# c_source reset +# --------------------------------------------------------------------------- # + +def test_c_source_zeroed(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-5.0]], exch_rxn_ids=["EX_glc"], + ) + apply_flux_data_constraints( + model, fd, bio_rxn_id="biomass", c_source="EX_eth", + ) + eth_rxn = model.reactions.get_by_id("EX_eth") + assert eth_rxn.lower_bound == 0.0 + assert eth_rxn.upper_bound == 0.0 + + +# --------------------------------------------------------------------------- # +# Loose mode +# --------------------------------------------------------------------------- # + +def test_loose_negative_flux_caps_lb(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-5.0]], exch_rxn_ids=["EX_glc"], + ) + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", loose_strict_flux="loose") + glc_rxn = model.reactions.get_by_id("EX_glc") + assert glc_rxn.lower_bound == -5.0 + assert glc_rxn.upper_bound == 0.0 + + +def test_loose_positive_flux_caps_ub(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[3.0]], exch_rxn_ids=["EX_eth"], + ) + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", loose_strict_flux="loose") + eth_rxn = model.reactions.get_by_id("EX_eth") + assert eth_rxn.lower_bound == 0.0 + assert eth_rxn.upper_bound == 3.0 + + +def test_loose_nan_flux_skipped(): + model = _build_simple_model() + original_lb = model.reactions.get_by_id("EX_O2").lower_bound + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[np.nan]], exch_rxn_ids=["EX_O2"], + ) + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass") + # NaN means "no measurement"; bounds untouched. + assert model.reactions.get_by_id("EX_O2").lower_bound == original_lb + + +# --------------------------------------------------------------------------- # +# Percentage variance mode +# --------------------------------------------------------------------------- # + +def test_percentage_brackets_positive_value(): + """pct=10 -> lb = val*0.95, ub = val*1.05.""" + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[3.0]], exch_rxn_ids=["EX_eth"], + ) + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", loose_strict_flux=10) + eth_rxn = model.reactions.get_by_id("EX_eth") + assert eth_rxn.lower_bound == pytest.approx(2.85) + assert eth_rxn.upper_bound == pytest.approx(3.15) + + +def test_percentage_handles_negative_value(): + """For negative val, lb and ub are swapped to keep lb <= ub.""" + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-5.0]], exch_rxn_ids=["EX_glc"], + ) + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", loose_strict_flux=10) + glc_rxn = model.reactions.get_by_id("EX_glc") + # val=-5, pct=10 -> raw [-5*0.95, -5*1.05] = [-4.75, -5.25] + # After swap: lb = -5.25, ub = -4.75. + assert glc_rxn.lower_bound == pytest.approx(-5.25) + assert glc_rxn.upper_bound == pytest.approx(-4.75) + + +# --------------------------------------------------------------------------- # +# Extreme +/-1000 sentinel +# --------------------------------------------------------------------------- # + +def test_minus_thousand_is_unconstrained_uptake(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-1000.0]], exch_rxn_ids=["EX_O2"], + ) + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", loose_strict_flux=10) + o2_rxn = model.reactions.get_by_id("EX_O2") + assert o2_rxn.lower_bound == -1000.0 + assert o2_rxn.upper_bound == 0.0 + + +def test_plus_thousand_is_unconstrained_excretion(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[1000.0]], exch_rxn_ids=["EX_eth"], + ) + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", loose_strict_flux=10) + eth_rxn = model.reactions.get_by_id("EX_eth") + assert eth_rxn.lower_bound == 0.0 + assert eth_rxn.upper_bound == 1000.0 + + +# --------------------------------------------------------------------------- # +# Condition selection +# --------------------------------------------------------------------------- # + +def test_condition_by_index(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1", "c2"], + p_tot=[0.5, 0.6], + gr_rate=[0.4, 0.7], + exch_fluxes=[[-5.0], [-8.0]], + exch_rxn_ids=["EX_glc"], + ) + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", condition=1) + bio = model.reactions.get_by_id("biomass") + assert bio.upper_bound == pytest.approx(0.7) + glc_rxn = model.reactions.get_by_id("EX_glc") + assert glc_rxn.lower_bound == -8.0 + + +def test_condition_by_name(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1", "c2"], + p_tot=[0.5, 0.6], + gr_rate=[0.4, 0.7], + exch_fluxes=[[-5.0], [-8.0]], + exch_rxn_ids=["EX_glc"], + ) + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", condition="c2") + glc_rxn = model.reactions.get_by_id("EX_glc") + assert glc_rxn.lower_bound == -8.0 + + +# --------------------------------------------------------------------------- # +# Multi-rxn flux scenarios +# --------------------------------------------------------------------------- # + +def test_multiple_rxns_constrained_correctly(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-5.0, -2.0, 3.0]], + exch_rxn_ids=["EX_glc", "EX_O2", "EX_eth"], + ) + apply_flux_data_constraints(model, fd, bio_rxn_id="biomass", loose_strict_flux="loose") + glc = model.reactions.get_by_id("EX_glc") + o2 = model.reactions.get_by_id("EX_O2") + eth = model.reactions.get_by_id("EX_eth") + assert (glc.lower_bound, glc.upper_bound) == (-5.0, 0.0) + assert (o2.lower_bound, o2.upper_bound) == (-2.0, 0.0) + assert (eth.lower_bound, eth.upper_bound) == (0.0, 3.0) + + +def test_missing_c_source_is_ignored(): + model = _build_simple_model() + fd = _flux_data( + conds=["c1"], p_tot=[0.5], gr_rate=[0.4], + exch_fluxes=[[-5.0]], exch_rxn_ids=["EX_glc"], + ) + apply_flux_data_constraints( + model, fd, bio_rxn_id="biomass", c_source="EX_absent", + ) + assert model.reactions.get_by_id("EX_glc").lower_bound == -5.0 diff --git a/tests/test_experiments_data.py b/tests/test_experiments_data.py new file mode 100644 index 0000000..9a53c04 --- /dev/null +++ b/tests/test_experiments_data.py @@ -0,0 +1,66 @@ +"""Tests for experiments.data.""" +from raven_toolbox.experiments import ( + ExperimentData, + load_experiment_data, +) + + +_FLUX_DATA_TSV = ( + "Condition\tPtot\tgrRate\tglucose (r_1714)\tethanol (r_1761)\t" + "weight\tsource\n" + "glucose\tNaN\t0.36\t-13.33\t19.83\t1\tDLKcat\n" + "ethanol\tNaN\t0.12\tNaN\t-8.5\t1\tbrenda\n" +) + +_MAX_GROWTH_TSV = ( + "Condition\tPtot\tgrRate\tglucose (r_1714)\tfructose (r_1709)\t" + "weight\tsource\n" + "glucose\tNaN\t0.41\t-1000\tNaN\t1\tDLKcat\n" + "fructose\tNaN\t0.338\tNaN\t-1000\t1\tDLKcat\n" +) + +_ZERO_EXCH_TSV = "Rxns\nr_1542\nr_1545\nr_1546\n" + + +def test_load_experiment_data_all_files_present(tmp_path): + data_dir = tmp_path + (data_dir / "fluxData.tsv").write_text(_FLUX_DATA_TSV) + (data_dir / "maxGrowth.tsv").write_text(_MAX_GROWTH_TSV) + (data_dir / "zeroFlux.tsv").write_text(_ZERO_EXCH_TSV) + + experiment_data = load_experiment_data(tmp_path) + + assert isinstance(experiment_data, ExperimentData) + + assert experiment_data.flux_data is not None + assert experiment_data.flux_data.conds == ["glucose", "ethanol"] + assert experiment_data.flux_data.exch_rxn_ids == ["r_1714", "r_1761"] + assert experiment_data.flux_data.source == ["DLKcat", "brenda"] + assert list(experiment_data.flux_data.weight) == [1.0, 1.0] + + assert experiment_data.max_growth is not None + assert experiment_data.max_growth.conds == ["glucose", "fructose"] + assert experiment_data.max_growth.exch_rxn_ids == ["r_1714", "r_1709"] + + assert experiment_data.zero_flux == ["r_1542", "r_1545", "r_1546"] + + +def test_load_experiment_data_all_files_missing(tmp_path): + + experiment_data = load_experiment_data(tmp_path) + + assert experiment_data.flux_data is None + assert experiment_data.max_growth is None + assert experiment_data.zero_flux == [] + + +def test_load_experiment_data_partial(tmp_path): + """Only flux_data present: max_growth stays None.""" + data_dir = tmp_path + (data_dir / "fluxData.tsv").write_text(_FLUX_DATA_TSV) + + experiment_data = load_experiment_data(tmp_path) + + assert experiment_data.flux_data is not None + assert experiment_data.max_growth is None + assert experiment_data.zero_flux == [] diff --git a/tests/test_experiments_flux_data.py b/tests/test_experiments_flux_data.py new file mode 100644 index 0000000..d34590e --- /dev/null +++ b/tests/test_experiments_flux_data.py @@ -0,0 +1,283 @@ +"""Tests for load_flux_data.""" +from pathlib import Path + +import numpy as np +import pytest + +from raven_toolbox.experiments import FluxData, load_flux_data + + +# --------------------------------------------------------------------------- # +# Fixture builders +# --------------------------------------------------------------------------- # + +def _write_tsv(path: Path, rows: list[list[str]]) -> None: + path.write_text( + "\n".join("\t".join(r) for r in rows) + "\n", + encoding="utf-8", + ) + + +# --------------------------------------------------------------------------- # +# Pre-flight +# --------------------------------------------------------------------------- # + +def test_missing_file_raises(tmp_path): + with pytest.raises(FileNotFoundError, match="fluxData file"): + load_flux_data(tmp_path / "missing.tsv") + + +def test_empty_file_raises(tmp_path): + p = tmp_path / "empty.tsv" + p.write_text("", encoding="utf-8") + with pytest.raises(ValueError, match="empty"): + load_flux_data(p) + + +def test_short_header_raises(tmp_path): + """Header with fewer than 3 columns -> error.""" + p = tmp_path / "short.tsv" + _write_tsv(p, [["only", "two"]]) + with pytest.raises(ValueError, match="header has"): + load_flux_data(p) + + +def test_no_exchange_columns_raises(tmp_path): + """Header with exactly 3 columns (no exchanges) -> error.""" + p = tmp_path / "no_exch.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate"], + ["c1", "0.5", "0.4"], + ]) + with pytest.raises(ValueError, match="no exchange-flux columns"): + load_flux_data(p) + + +# --------------------------------------------------------------------------- # +# Basic happy path +# --------------------------------------------------------------------------- # + +def test_old_weight_column_name_is_rejected(tmp_path): + f = tmp_path / "fluxData.tsv" + _write_tsv(f, [ + ["Condition", "Ptot", "grRate", "glucose (r_1714)", "evotuneRMSEweight"], + ["c1", "0.5", "0.4", "-10", "1"], + ]) + with pytest.raises(ValueError, match="now called 'weight'"): + load_flux_data(f) + + +def test_weight_column_is_read_and_removed_from_the_fluxes(tmp_path): + f = tmp_path / "fluxData.tsv" + _write_tsv(f, [ + ["Condition", "Ptot", "grRate", "glucose (r_1714)", "weight", "source"], + ["c1", "0.5", "0.4", "-10", "0", "x"], + ["c2", "0.5", "0.3", "-8", "2", "y"], + ]) + data = load_flux_data(f) + assert data.weight.tolist() == [0.0, 2.0] + assert data.exch_rxn_ids == ["r_1714"] + + +def test_basic_single_condition(tmp_path): + p = tmp_path / "fd.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", "glucose (EX_glc)"], + ["c1", "0.5", "0.4", "-10.0"], + ]) + fd = load_flux_data(p) + assert isinstance(fd, FluxData) + assert fd.conds == ["c1"] + np.testing.assert_array_equal(fd.p_tot, [0.5]) + np.testing.assert_array_equal(fd.gr_rate, [0.4]) + np.testing.assert_array_equal(fd.exch_fluxes, [[-10.0]]) + assert fd.exch_mets == ["glucose"] + assert fd.exch_rxn_ids == ["EX_glc"] + + +def test_multi_condition_multi_exchange(tmp_path): + p = tmp_path / "fd.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", + "glucose (EX_glc)", "oxygen (EX_O2)", "ethanol (EX_eth)"], + ["c1", "0.5", "0.4", "-10.0", "-2.0", "5.0"], + ["c2", "0.6", "0.5", "-8.5", "-3.0", "7.0"], + ]) + fd = load_flux_data(p) + assert fd.conds == ["c1", "c2"] + np.testing.assert_array_equal(fd.p_tot, [0.5, 0.6]) + np.testing.assert_array_equal(fd.gr_rate, [0.4, 0.5]) + np.testing.assert_array_equal( + fd.exch_fluxes, + [[-10.0, -2.0, 5.0], [-8.5, -3.0, 7.0]], + ) + assert fd.exch_mets == ["glucose", "oxygen", "ethanol"] + assert fd.exch_rxn_ids == ["EX_glc", "EX_O2", "EX_eth"] + + +# --------------------------------------------------------------------------- # +# Exchange header parsing +# --------------------------------------------------------------------------- # + +def test_exchange_header_with_spaces_in_met_name(tmp_path): + p = tmp_path / "fd.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", "carbon dioxide (EX_co2)"], + ["c1", "0.5", "0.4", "5.0"], + ]) + fd = load_flux_data(p) + assert fd.exch_mets == ["carbon dioxide"] + assert fd.exch_rxn_ids == ["EX_co2"] + + +def test_exchange_header_without_parens_passes_through(tmp_path): + """Header without `(...)` falls back to met=rxn=full header.""" + p = tmp_path / "fd.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", "EX_glc"], + ["c1", "0.5", "0.4", "-10.0"], + ]) + fd = load_flux_data(p) + assert fd.exch_mets == ["EX_glc"] + assert fd.exch_rxn_ids == ["EX_glc"] + + +# --------------------------------------------------------------------------- # +# NaN handling +# --------------------------------------------------------------------------- # + +def test_empty_flux_cell_yields_nan(tmp_path): + p = tmp_path / "fd.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", "glc (EX_glc)", "o2 (EX_O2)"], + ["c1", "0.5", "0.4", "-10.0", ""], + ["c2", "0.6", "0.5", "", "-3.0"], + ]) + fd = load_flux_data(p) + assert np.isnan(fd.exch_fluxes[0, 1]) + assert np.isnan(fd.exch_fluxes[1, 0]) + assert fd.exch_fluxes[0, 0] == -10.0 + assert fd.exch_fluxes[1, 1] == -3.0 + + +def test_non_numeric_flux_yields_nan(tmp_path): + p = tmp_path / "fd.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", "glc (EX_glc)"], + ["c1", "0.5", "0.4", "NA"], + ]) + fd = load_flux_data(p) + assert np.isnan(fd.exch_fluxes[0, 0]) + + +# --------------------------------------------------------------------------- # +# Optional columns +# --------------------------------------------------------------------------- # + +def test_no_optional_columns_yields_none(tmp_path): + p = tmp_path / "fd.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", "glc (EX_glc)"], + ["c1", "0.5", "0.4", "-10.0"], + ]) + fd = load_flux_data(p) + assert fd.weight is None + assert fd.source is None + + +def test_weight_column_loaded(tmp_path): + p = tmp_path / "fd.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", + "glc (EX_glc)", "weight"], + ["c1", "0.5", "0.4", "-10.0", "1.5"], + ["c2", "0.6", "0.5", "-8.5", "0.7"], + ]) + fd = load_flux_data(p) + assert fd.weight is not None + np.testing.assert_array_equal(fd.weight, [1.5, 0.7]) + # Tuning-weight column dropped from exch_fluxes. + assert fd.exch_fluxes.shape == (2, 1) + assert fd.exch_rxn_ids == ["EX_glc"] + + +def test_source_column_loaded(tmp_path): + p = tmp_path / "fd.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", "glc (EX_glc)", "source"], + ["c1", "0.5", "0.4", "-10.0", "PMID:12345"], + ["c2", "0.6", "0.5", "-8.5", "in-house"], + ]) + fd = load_flux_data(p) + assert fd.source == ["PMID:12345", "in-house"] + assert fd.exch_fluxes.shape == (2, 1) + + +def test_both_optional_columns(tmp_path): + p = tmp_path / "fd.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", + "glc (EX_glc)", "weight", "source"], + ["c1", "0.5", "0.4", "-10.0", "1.5", "PMID:12345"], + ]) + fd = load_flux_data(p) + assert fd.weight is not None + np.testing.assert_array_equal(fd.weight, [1.5]) + assert fd.source == ["PMID:12345"] + assert fd.exch_fluxes.shape == (1, 1) + + +def test_optional_columns_in_arbitrary_position(tmp_path): + """Optional columns can appear before, between, or after the + exchange columns; they are pulled out independently of position.""" + p = tmp_path / "fd.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", + "source", "glc (EX_glc)", "weight"], + ["c1", "0.5", "0.4", "PMID:1", "-10.0", "1.5"], + ]) + fd = load_flux_data(p) + assert fd.source == ["PMID:1"] + assert fd.weight is not None + np.testing.assert_array_equal(fd.weight, [1.5]) + np.testing.assert_array_equal(fd.exch_fluxes, [[-10.0]]) + assert fd.exch_rxn_ids == ["EX_glc"] + + +# --------------------------------------------------------------------------- # +# Blank lines +# --------------------------------------------------------------------------- # + +def test_blank_lines_skipped(tmp_path): + p = tmp_path / "fd.tsv" + p.write_text( + "condition\tPtot\tgrRate\tglc (EX_glc)\n" + "\n" + "c1\t0.5\t0.4\t-10.0\n" + "\n", + encoding="utf-8", + ) + fd = load_flux_data(p) + assert fd.conds == ["c1"] + + +def test_ragged_row_raises(tmp_path): + p = tmp_path / "ragged.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", "Glc (EX_a)", "O2 (EX_b)"], + ["c1", "0.5", "0.4", "-1", "2"], + ["c2", "0.5", "0.4", "-1"], + ]) + with pytest.raises(ValueError, match="row 3 has 4 cell"): + load_flux_data(p) + + +def test_nested_parentheses_in_reaction_id(tmp_path): + p = tmp_path / "nested.tsv" + _write_tsv(p, [ + ["condition", "Ptot", "grRate", "X (EX_a(e))", "O2 (EX_b)"], + ["c1", "0.5", "0.4", "-1", "2"], + ]) + fd = load_flux_data(p) + assert fd.exch_mets == ["X", "O2"] + assert fd.exch_rxn_ids == ["EX_a(e)", "EX_b"] diff --git a/tests/test_utils_optimize_model.py b/tests/test_utils_optimize_model.py new file mode 100644 index 0000000..06804da --- /dev/null +++ b/tests/test_utils_optimize_model.py @@ -0,0 +1,62 @@ +"""Tests for optimize_model.""" +import math + +import cobra +import pytest + +from raven_toolbox.utils.solver import optimize_model + + +def _infeasible_model(solver: str) -> cobra.Model: + model = cobra.Model("m") + a = cobra.Metabolite("A", compartment="c") + rxn = cobra.Reaction("R1") + rxn.add_metabolites({a: 1.0}) + rxn.bounds = (0.0, 10.0) + sink = cobra.Reaction("S1") + sink.add_metabolites({a: -1.0}) + sink.bounds = (20.0, 30.0) + model.add_reactions([rxn, sink]) + model.objective = "R1" + model.solver = solver + return model + + +@pytest.mark.parametrize("solver", ["glpk", "scipy"]) +def test_infeasible_lp_gives_an_infeasible_solution(solver): + solution = optimize_model(_infeasible_model(solver)) + assert solution.status == "infeasible" + + +def test_feasible_lp_matches_model_optimize(): + model = _infeasible_model("glpk") + model.reactions.get_by_id("S1").bounds = (0.0, 5.0) + solution = optimize_model(model) + assert solution.status == "optimal" + assert solution.objective_value == pytest.approx(5.0) + + +def test_gurobi_presolved_infeasible_lp_gives_an_infeasible_solution(): + """Gurobi holds no primal values for an infeasible LP that it presolved, which makes + ``model.optimize()`` raise ``GurobiError``.""" + try: + model = _infeasible_model("gurobi") + except Exception as exc: + pytest.skip(f"gurobi unusable: {exc}") + model.solver.problem.Params.Presolve = 2 + solution = optimize_model(model) + assert solution.status == "infeasible" + assert math.isnan(solution.objective_value) + assert set(solution.fluxes.index) == {"R1", "S1"} + + +def test_other_errors_are_raised(): + class Broken: + class solver: + status = "optimal" + + def optimize(self, **kwargs): + raise RuntimeError("boom") + + with pytest.raises(RuntimeError, match="boom"): + optimize_model(Broken()) diff --git a/tests/test_utils_protein.py b/tests/test_utils_protein.py new file mode 100644 index 0000000..0364543 --- /dev/null +++ b/tests/test_utils_protein.py @@ -0,0 +1,168 @@ +"""Tests for calculate_mw.""" +import logging +import math + +import pytest + +from raven_toolbox.utils.protein import ( + calculate_mw, + _RESIDUE_MASSES, + _STANDARD_RESIDUE_MASSES, + _WATER_MASS, + _X_MASS, +) + + +# --------------------------------------------------------------------------- # +# Trivial cases +# --------------------------------------------------------------------------- # + +def test_empty_sequence_returns_nan(): + # An empty / all-skipped sequence has no residues; NaN avoids being + # mistaken for an 18 Da "protein". + assert math.isnan(calculate_mw("")) + assert math.isnan(calculate_mw(" ")) + assert math.isnan(calculate_mw("123")) + + +def test_single_residue(): + assert calculate_mw("A") == pytest.approx(_WATER_MASS + 71.08) + + +def test_dipeptide(): + expected = _WATER_MASS + 71.08 + 137.14 + assert calculate_mw("AH") == pytest.approx(expected) + + +# --------------------------------------------------------------------------- # +# Known reference cases +# --------------------------------------------------------------------------- # + +def test_methionine_only(): + """Met has mass 131.20.""" + assert calculate_mw("M") == pytest.approx(_WATER_MASS + 131.20) + + +def test_repeated_residues_count_correctly(): + expected = _WATER_MASS + 5 * 71.08 + assert calculate_mw("AAAAA") == pytest.approx(expected) + + +def test_all_20_standard_residues(): + """A sequence of one of each of the 20 standard residues should + give water + sum of all 20 masses.""" + seq = "".join(sorted(_STANDARD_RESIDUE_MASSES.keys())) + expected = _WATER_MASS + sum(_STANDARD_RESIDUE_MASSES.values()) + assert calculate_mw(seq) == pytest.approx(expected) + + +# --------------------------------------------------------------------------- # +# Non-standard codes +# --------------------------------------------------------------------------- # + +def test_b_is_average_of_d_and_n(): + expected = (_RESIDUE_MASSES["D"] + _RESIDUE_MASSES["N"]) / 2 + assert _RESIDUE_MASSES["B"] == pytest.approx(expected) + + +def test_z_is_average_of_e_and_q(): + expected = (_RESIDUE_MASSES["E"] + _RESIDUE_MASSES["Q"]) / 2 + assert _RESIDUE_MASSES["Z"] == pytest.approx(expected) + + +def test_j_equals_l_and_i(): + assert _RESIDUE_MASSES["J"] == _RESIDUE_MASSES["L"] == _RESIDUE_MASSES["I"] + + +def test_o_pyrrolysine(): + assert _RESIDUE_MASSES["O"] == pytest.approx(255.31) + + +def test_u_selenocysteine(): + assert _RESIDUE_MASSES["U"] == pytest.approx(150.04) + + +def test_x_is_mean_of_20_standard_residues(): + expected = sum(_STANDARD_RESIDUE_MASSES.values()) / 20 + assert _X_MASS == pytest.approx(expected) + assert _X_MASS == pytest.approx(118.885, abs=1e-3) + + +# --------------------------------------------------------------------------- # +# Case insensitivity +# --------------------------------------------------------------------------- # + +def test_lowercase_is_treated_same_as_uppercase(): + assert calculate_mw("ahkc") == calculate_mw("AHKC") + + +def test_mixed_case_works(): + assert calculate_mw("AhKc") == calculate_mw("AHKC") + + +# --------------------------------------------------------------------------- # +# Whitespace and digits ignored silently +# --------------------------------------------------------------------------- # + +def test_whitespace_ignored(): + assert calculate_mw("A H") == calculate_mw("AH") + assert calculate_mw("\tA\nH\n") == calculate_mw("AH") + + +def test_digits_ignored(): + assert calculate_mw("A1H2") == calculate_mw("AH") + + +def test_whitespace_and_digits_do_not_warn(caplog): + with caplog.at_level(logging.WARNING): + calculate_mw("A H 1 2 K") + assert "skipped" not in caplog.text + + +# --------------------------------------------------------------------------- # +# Unknown characters: warn but do not raise +# --------------------------------------------------------------------------- # + +def test_unknown_character_warns(caplog): + with caplog.at_level(logging.WARNING): + calculate_mw("A@H") + assert "skipped" in caplog.text + assert "@" in caplog.text + + +def test_unknown_character_does_not_contribute_to_mass(caplog): + with caplog.at_level(logging.WARNING): + result = calculate_mw("A@H") + assert result == calculate_mw("AH") + + +def test_multiple_unknown_chars_warned_together(caplog): + with caplog.at_level(logging.WARNING): + calculate_mw("A@H#K?L") + assert "skipped" in caplog.text + assert "@" in caplog.text + assert "#" in caplog.text + assert "?" in caplog.text + + +# --------------------------------------------------------------------------- # +# Realistic sequence +# --------------------------------------------------------------------------- # + +def test_short_real_protein_mass(): + """A snippet of a real protein sequence should yield a plausible + mass in the low-kDa range, not just any positive number.""" + seq = "MENTGTLAGCDVRFGRSGGSGSGSGSAATAACK" + mw = calculate_mw(seq) + assert mw > 0 + # Approximately 33 residues at average ~110 g/mol = 3630 + water. + assert 3000 < mw < 4500 + + +# --------------------------------------------------------------------------- # +# Water mass precision +# --------------------------------------------------------------------------- # + +def test_water_mass_is_precise(): + """The water mass used is the precise average, 18.01528 Da.""" + assert _WATER_MASS == pytest.approx(18.01528) diff --git a/tests/test_utils_solver_scipy.py b/tests/test_utils_solver_scipy.py new file mode 100644 index 0000000..23d2ae5 --- /dev/null +++ b/tests/test_utils_solver_scipy.py @@ -0,0 +1,256 @@ +"""Tests for the scipy (HiGHS) support in raven_toolbox.utils.solver. + +optlang's scipy interface keeps a variable index map that goes wrong after a removal, +which makes cobra read fluxes from the wrong columns, and it raises for duals. +raven_toolbox.utils.solver repairs the first and reads the HiGHS marginals for the second. +""" +import cobra +import numpy as np +import pytest + +from raven_toolbox.utils.solver import ( + constraint_shadow_prices, + edit_on_fast_solver, + repair_scipy_problem, + switch_to_scipy, +) + +@pytest.fixture +def scipy_default(): + """cobra builds every new model on scipy while the test runs.""" + config = cobra.Configuration() + previous = config.solver + config.solver = "scipy" + yield + config.solver = previous + + +def _columns_are_consistent(model): + problem = model.solver.problem + return list(problem._variables.values()) == list(range(len(problem._variables))) + + +def _ring(solver): + """A plain cobra model on ``solver`` with more variables than constraints.""" + config = cobra.Configuration() + previous = config.solver + config.solver = solver + try: + model = cobra.Model("plain") + mets = [cobra.Metabolite(f"m{i}") for i in range(3)] + rxns = [] + for i in range(6): + rxn = cobra.Reaction(f"r{i}") + rxn.bounds = (0, 10) + rxn.add_metabolites({mets[i % 3]: 1.0, mets[(i + 1) % 3]: -1.0} if i else {mets[0]: 1.0}) + rxns.append(rxn) + model.add_reactions(rxns) + finally: + config.solver = previous + model.objective = "r5" + return model + + +def test_a_plain_cobra_model_is_repaired_on_request(): + model, reference = _ring("scipy"), _ring("glpk") + for m in (model, reference): + m.remove_reactions(["r1", "r2"]) + m.reactions.get_by_id("r4").bounds = (0, 3) + assert repair_scipy_problem(model) + assert _columns_are_consistent(model) + assert model.slim_optimize() == pytest.approx(reference.slim_optimize()) + assert model.reactions.get_by_id("r5").flux == pytest.approx(reference.reactions.get_by_id("r5").flux) + + +def test_repair_does_nothing_on_other_solvers(): + model = cobra.Model("m") + model.solver = "glpk" + assert repair_scipy_problem(model) is False + + +def _columns_ring(solver): + """Three metabolites, twelve reactions: removals leave late variables with stale columns.""" + config = cobra.Configuration() + previous = config.solver + config.solver = solver + try: + model = cobra.Model("ring") + mets = [cobra.Metabolite(f"m{i}") for i in range(3)] + rxns = [] + for i in range(12): + rxn = cobra.Reaction(f"r{i}") + rxn.bounds = (0, 10 + i) + if i == 0: + rxn.add_metabolites({mets[0]: 1.0}) + else: + rxn.add_metabolites({mets[i % 3]: -1.0, mets[(i + 1) % 3]: 0.5 + 0.1 * i}) + rxns.append(rxn) + model.add_reactions(rxns) + finally: + config.solver = previous + model.objective = "r11" + return model + + +def test_removals_after_the_install_keep_every_column_right(): + model, reference = _columns_ring("scipy"), _columns_ring("glpk") + assert repair_scipy_problem(model) + for rid in ("r3", "r7", "r9", "r10"): + model.reactions.get_by_id(rid).delete() + reference.reactions.get_by_id(rid).delete() + assert _columns_are_consistent(model) + assert model.slim_optimize() == pytest.approx(reference.slim_optimize()) + solution = model.optimize() + expected = reference.optimize() + assert solution.objective_value == pytest.approx(expected.objective_value) + for rid in solution.fluxes.index: + assert model.reactions.get_by_id(rid).flux == pytest.approx(solution.fluxes[rid]) + + +def test_a_map_that_is_already_stale_is_rebuilt_from_the_expressions(): + model, reference = _columns_ring("scipy"), _columns_ring("glpk") + model.reactions.get_by_id("r5").delete() + reference.reactions.get_by_id("r5").delete() + model.solver.update() + assert not _columns_are_consistent(model) + assert repair_scipy_problem(model) + assert _columns_are_consistent(model) + assert model.slim_optimize() == pytest.approx(reference.slim_optimize()) + + +def test_the_sparse_problem_hands_highs_the_matrix_of_the_dense_problem(): + model = _columns_ring("scipy") + dense = model.solver.problem.A.copy() + assert repair_scipy_problem(model) + assert (model.solver.problem.A.toarray() == dense).all() + + +def test_switch_to_scipy_keeps_the_objective_and_its_direction(): + model, reference = _columns_ring("glpk"), _columns_ring("glpk") + for m in (model, reference): + m.objective = "r8" + m.objective_direction = "min" + m.reactions.get_by_id("r1").lower_bound = 2.0 + switch_to_scipy(model) + assert model.solver.interface.__name__.endswith("scipy_interface") + assert model.objective_direction == "min" + assert model.slim_optimize() == pytest.approx(reference.slim_optimize()) + assert model.reactions.get_by_id("r8").flux == pytest.approx(model.slim_optimize()) + + +@pytest.mark.parametrize("status_case", ["infeasible", "unbounded"]) +def test_a_solve_without_optimum_returns_a_solution_with_its_status(status_case): + config = cobra.Configuration() + previous = config.solver + config.solver = "scipy" + try: + model = cobra.Model("chain") + met = cobra.Metabolite("m") + source, sink = cobra.Reaction("source"), cobra.Reaction("sink") + source.add_metabolites({met: 1.0}) + sink.add_metabolites({met: -1.0}) + source.bounds = (0, 10) if status_case == "infeasible" else (0, 1000.0) + sink.bounds = (20, 30) if status_case == "infeasible" else (0, 1000.0) + model.add_reactions([source, sink]) + finally: + config.solver = previous + model.objective = "sink" + if status_case == "unbounded": + source.upper_bound = float("inf") + sink.upper_bound = float("inf") + assert repair_scipy_problem(model) + if status_case == "unbounded": + # cobra raises for this status on every solver + with pytest.raises(cobra.exceptions.OptimizationError, match="unbounded"): + model.optimize(raise_error=False) + return + solution = model.optimize(raise_error=False) + assert solution.status == status_case + assert np.isnan(solution.objective_value) + assert solution.fluxes.isna().all() + + +def test_removing_several_constraints_at_once_on_the_sparse_problem(): + model = _columns_ring("scipy") + repair_scipy_problem(model) + ids = [m.id for m in list(model.metabolites)[:3]] + model.remove_cons_vars([model.constraints[i] for i in ids]) + model.solver.update() + assert len(model.solver.problem._constraints) == 2 * len(model.constraints) + assert not set(ids) & set(model.constraints) + + +def test_edit_on_fast_solver_returns_to_scipy_with_the_same_problem(): + model = _columns_ring("scipy") + repair_scipy_problem(model) + before = model.slim_optimize() + with edit_on_fast_solver(model) as fast: + assert fast + assert model.solver.interface.__name__.endswith("glpk_interface") + model.reactions.get_by_id("r1").bounds = (0.0, 5.0) + assert model.solver.interface.__name__.endswith("scipy_interface") + assert model.reactions.get_by_id("r1").upper_bound == 5.0 + assert model.slim_optimize() <= before + reference = _columns_ring("glpk") + reference.reactions.get_by_id("r1").bounds = (0.0, 5.0) + assert model.slim_optimize() == pytest.approx(reference.slim_optimize()) + + +def test_edit_on_fast_solver_changes_nothing_on_glpk(): + model = _columns_ring("glpk") + with edit_on_fast_solver(model) as fast: + assert not fast + assert model.solver.interface.__name__.endswith("glpk_interface") + + +def test_model_tolerance_reaches_highs_and_survives_a_copy(monkeypatch): + import pickle + + from scipy import optimize + + model = _ring("scipy") + repair_scipy_problem(model) + model.tolerance = 1e-9 + assert model.solver.configuration.tolerances.feasibility == 1e-9 + assert model.solver.configuration.tolerances.optimality == 1e-9 + + seen = [] + real = optimize.linprog + + def spy(*args, **kwargs): + seen.append(kwargs["options"]) + return real(*args, **kwargs) + + for m in (model, pickle.loads(pickle.dumps(model)), model.copy()): + monkeypatch.setattr("raven_toolbox.utils.solver.linprog", spy) + m.slim_optimize() + assert len(seen) == 3 + assert all(o["primal_feasibility_tolerance"] == 1e-9 for o in seen) + assert all(o["dual_feasibility_tolerance"] == 1e-9 for o in seen) + + +def test_a_solve_with_numerical_difficulties_is_repeated_at_the_default_tolerance(monkeypatch): + from scipy import optimize + + model = _ring("scipy") + repair_scipy_problem(model) + expected = model.slim_optimize() + model.tolerance = 1e-9 + real = optimize.linprog + calls = [] + + def numerical_trouble_when_tight(*args, **kwargs): + calls.append(kwargs["options"]) + if "primal_feasibility_tolerance" in kwargs["options"]: + return optimize.OptimizeResult(status=4, fun=None, x=None, slack=None) + return real(*args, **kwargs) + + monkeypatch.setattr("raven_toolbox.utils.solver.linprog", numerical_trouble_when_tight) + assert model.slim_optimize() == pytest.approx(expected) + assert len(calls) == 2 + + def always_numerical_trouble(*args, **kwargs): + return optimize.OptimizeResult(status=4, fun=None, x=None, slack=None) + + monkeypatch.setattr("raven_toolbox.utils.solver.linprog", always_numerical_trouble) + assert model.optimize(raise_error=False).status == "numeric"