diff --git a/src/mavedb/lib/acmg.py b/src/mavedb/lib/acmg.py index d7de860e8..d786e6a96 100644 --- a/src/mavedb/lib/acmg.py +++ b/src/mavedb/lib/acmg.py @@ -8,6 +8,48 @@ from mavedb.models.enums.strength_of_evidence import StrengthOfEvidenceProvided +def acmg_evidence_outcome_code(criterion: str, evidence_strength: Optional[str]) -> str: + """Build the ACMG 2015 evidence outcome code for a criterion and the strength it was met at. + + Three rules, which are the ACMG convention rather than anything MaveDB invented: + + - no strength means the criterion was evaluated and *not* met, written ``"PS3_not_met"`` + - STRONG is the criterion's baseline, so it is written bare: ``"PS3"`` + - any other strength is suffixed: ``"PS3_moderate"`` + + Takes the criterion code and strength *name* as strings rather than enums so that the VA-Spec + annotation builders and the flat exports can share one implementation despite drawing their + enumerations from different places. That also means this survives any future decision about which + enumeration is canonical. + + Parameters + ---------- + criterion : str + The criterion code, e.g. ``"PS3"`` or ``"BS3"``. + evidence_strength : Optional[str] + The strength name, e.g. ``"MODERATE"``. None when the criterion was not met. + + Returns + ------- + str + The evidence outcome code. + + Examples + -------- + >>> acmg_evidence_outcome_code("PS3", "STRONG") + 'PS3' + >>> acmg_evidence_outcome_code("PS3", "MODERATE") + 'PS3_moderate' + >>> acmg_evidence_outcome_code("BS3", None) + 'BS3_not_met' + """ + if evidence_strength is None: + return f"{criterion}_not_met" + if evidence_strength.upper() == StrengthOfEvidenceProvided.STRONG.name: + return criterion + return f"{criterion}_{evidence_strength.lower()}" + + def points_evidence_strength_equivalent( points: int, ) -> tuple[Optional[ACMGCriterion], Optional[StrengthOfEvidenceProvided]]: diff --git a/src/mavedb/lib/annotation/classification.py b/src/mavedb/lib/annotation/classification.py index 08fc3b208..c7707eb1c 100644 --- a/src/mavedb/lib/annotation/classification.py +++ b/src/mavedb/lib/annotation/classification.py @@ -21,13 +21,35 @@ class ExperimentalVariantFunctionalImpactClassification(StrEnum): INDETERMINATE = "indeterminate" +def _classification_contains_variant( + functional_classification: ScoreCalibrationFunctionalClassification, + mapped_variant: MappedVariant, + containing_classification_ids: Optional[set[int]], +) -> bool: + """Whether this classification's score range contains the variant. + + Prefers a pre-resolved id set, which is an O(1) check. Falls back to the ORM relationship, which is + correct but loads every variant of the range. + """ + if containing_classification_ids is not None: + return functional_classification.id in containing_classification_ids + return mapped_variant.variant in functional_classification.variants + + def functional_classification_of_variant( - mapped_variant: MappedVariant, score_calibration: ScoreCalibration + mapped_variant: MappedVariant, + score_calibration: ScoreCalibration, + containing_classification_ids: Optional[set[int]] = None, ) -> tuple[Optional[ScoreCalibrationFunctionalClassification], ExperimentalVariantFunctionalImpactClassification]: """Classify a variant's functional impact as normal, abnormal, or indeterminate. Uses the primary score calibration and its functional ranges. Raises ValueError if required calibration or score is missing. + + *containing_classification_ids*, when given, is the set of functional-classification ids already known + to contain this variant. Pass it to avoid the ORM membership check below, which loads every variant of + every range. A caller classifying many variants should resolve membership once from the association + table; see ``mavedb.lib.csv.variant``. """ if not mapped_variant.variant.score_set.score_calibrations: raise ValueError( @@ -41,11 +63,8 @@ def functional_classification_of_variant( " Unable to classify functional impact." ) - # TODO#XXX: Performance: avoid ORM relationship membership checks (`variant in functional_range.variants`) in this - # DB-agnostic function. Resolve class-based matches in an upstream DB-aware layer using the association table, - # pass matched functional classification IDs into this function, and use O(1) ID membership checks here. for functional_range in score_calibration.functional_classifications: - if mapped_variant.variant in functional_range.variants: + if _classification_contains_variant(functional_range, mapped_variant, containing_classification_ids): if functional_range.functional_classification is FunctionalClassificationOptions.normal: return functional_range, ExperimentalVariantFunctionalImpactClassification.NORMAL elif functional_range.functional_classification is FunctionalClassificationOptions.abnormal: @@ -58,6 +77,7 @@ def functional_classification_of_variant( def pathogenicity_classification_of_variant( mapped_variant: MappedVariant, score_calibration: ScoreCalibration, + containing_classification_ids: Optional[set[int]] = None, ) -> tuple[ Optional[ScoreCalibrationFunctionalClassification], VariantPathogenicityEvidenceLine.Criterion, @@ -87,11 +107,8 @@ def pathogenicity_classification_of_variant( " Unable to classify clinical impact." ) - # TODO#XXX: Performance: avoid ORM relationship membership checks (`variant in pathogenicity_range.variants`) in this - # DB-agnostic function. Resolve class-based matches in an upstream DB-aware layer using the association table, - # pass matched functional classification IDs into this function, and use O(1) ID membership checks here. for pathogenicity_range in score_calibration.functional_classifications: - if mapped_variant.variant in pathogenicity_range.variants: + if _classification_contains_variant(pathogenicity_range, mapped_variant, containing_classification_ids): if pathogenicity_range.acmg_classification is None: return (pathogenicity_range, VariantPathogenicityEvidenceLine.Criterion.PS3, None) diff --git a/src/mavedb/lib/annotation/evidence_line.py b/src/mavedb/lib/annotation/evidence_line.py index 8ebf7f163..d30e90542 100644 --- a/src/mavedb/lib/annotation/evidence_line.py +++ b/src/mavedb/lib/annotation/evidence_line.py @@ -10,8 +10,8 @@ StudyResult, VariantPathogenicityProposition, ) -from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided +from mavedb.lib.acmg import acmg_evidence_outcome_code from mavedb.lib.annotation.classification import ( functional_classification_of_variant, pathogenicity_classification_of_variant, @@ -44,16 +44,14 @@ def acmg_evidence_line( mapped_variant, score_calibration ) + evidence_outcome_code = acmg_evidence_outcome_code( + evidence_outcome.value, evidence_strength.name if evidence_strength else None + ) + if not evidence_strength: - evidence_outcome_code = f"{evidence_outcome.value}_not_met" strength_of_evidence = None direction_of_evidence = Direction.NEUTRAL else: - evidence_outcome_code = ( - f"{evidence_outcome.value}_{evidence_strength.name.lower()}" - if evidence_strength != StrengthOfEvidenceProvided.STRONG - else evidence_outcome.value - ) strength_of_evidence = MappableConcept( primaryCoding=Coding( code=evidence_strength, diff --git a/src/mavedb/lib/annotation/flatten.py b/src/mavedb/lib/annotation/flatten.py new file mode 100644 index 000000000..04d2506f7 --- /dev/null +++ b/src/mavedb/lib/annotation/flatten.py @@ -0,0 +1,140 @@ +"""Flatten a variant's VA-Spec clinical interpretation into scalar values. + +The rest of this package builds nested VA-Spec structures, which are faithful to the standard but not +consumable by a spreadsheet. This projects the same classification onto flat scalars. +""" + +from dataclasses import dataclass +from typing import Optional + +from ga4gh.va_spec.acmg_2015 import AcmgClassification + +from mavedb.lib.acmg import acmg_evidence_outcome_code +from mavedb.lib.annotation.classification import ( + functional_classification_of_variant, + pathogenicity_classification_of_variant, +) +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration + + +@dataclass(frozen=True) +class FlatAnnotation: + """A single variant's clinical interpretation under one calibration, flattened to scalars. + + A field is ``None`` when the calibration cannot support it; exporters render that as NA. + """ + + functional_classification: Optional[str] = None + """"normal", "abnormal", or "indeterminate".""" + + acmg_criterion: Optional[str] = None + """The ACMG 2015 criterion evaluated, e.g. "PS3" or "BS3".""" + + acmg_evidence_strength: Optional[str] = None + """Strength of evidence, e.g. "MODERATE" or "MODERATE_PLUS"; None when the criterion was not met. + + MaveDB's own enumeration, finer-grained than VA-Spec's: an M+ range reports MODERATE_PLUS here while + its VA-Spec annotation must report moderate. + """ + + acmg_evidence_outcome_code: Optional[str] = None + """ACMG evidence outcome code, e.g. "PS3_moderate", "PS3" (strong), or "BS3_not_met".""" + + pathogenicity_classification: Optional[str] = None + """"PATHOGENIC", "BENIGN", or "UNCERTAIN_SIGNIFICANCE".""" + + calibration_urn: Optional[str] = None + calibration_title: Optional[str] = None + + research_use_only: Optional[bool] = None + """Whether the calibration is marked research use only. + + Carried so a consumer holding only exported rows can tell that a criterion came from thresholds never + validated for clinical use. + """ + + +def flatten_annotation( + mapped_variant: MappedVariant, + score_calibration: Optional[ScoreCalibration], + containing_classification_ids: Optional[set[int]] = None, +) -> FlatAnnotation: + """Flatten a variant's clinical interpretation under *score_calibration* into scalar values. + + A calibration with ranges but no ACMG classifications yields a functional classification only, matching + the annotation layer. Evidence strength uses MaveDB's own enumeration, so MODERATE_PLUS is preserved. + + Args: + containing_classification_ids: forwarded to the classifiers. Supply it when flattening many + variants; the fallback loads every variant of every score range. + + Returns: + An all-``None`` annotation when *score_calibration* is None. A calibration that exists but defines + no ranges reports its identity and standing with no interpretation. + """ + # No calibration means nothing to say under this namespace (e.g. it belongs to another score set). + if score_calibration is None: + return FlatAnnotation() + + # Rangeless: reporting identity distinguishes "defines no ranges" from "no calibration applies here". + if not score_calibration.functional_classifications: + return FlatAnnotation( + calibration_urn=score_calibration.urn, + calibration_title=score_calibration.title, + research_use_only=bool(score_calibration.research_use_only), + ) + + _, functional_classification = functional_classification_of_variant( + mapped_variant, score_calibration, containing_classification_ids + ) + + functional_only = FlatAnnotation( + functional_classification=functional_classification.value, + calibration_urn=score_calibration.urn, + calibration_title=score_calibration.title, + research_use_only=bool(score_calibration.research_use_only), + ) + + # No ACMG classification on any range: stop at the functional classification rather than reporting a + # not-met PS3 the curator never asserted. + if all(fc.acmg_classification is None for fc in score_calibration.functional_classifications): + return functional_only + + # VA-Spec strength deliberately unused: it has already collapsed MODERATE_PLUS to MODERATE. + containing_range, criterion, _va_spec_evidence_strength = pathogenicity_classification_of_variant( + mapped_variant, score_calibration, containing_classification_ids + ) + + # Read the strength off the containing range, which keeps MODERATE_PLUS. Reachable in practice: + # `points_evidence_strength_equivalent` assigns M+ to +/-3 point ranges (e.g. the Excalibr + # calibrations). + # + # TODO(#XXX): move the lossy MODERATE_PLUS -> MODERATE conversion to the VA-Spec boundary instead of + # `classification.py`, upstream of every consumer; this special case then disappears. + native_evidence_strength = ( + containing_range.acmg_classification.evidence_strength + if containing_range is not None and containing_range.acmg_classification is not None + else None + ) + evidence_strength_name = native_evidence_strength.name if native_evidence_strength is not None else None + + # `pathogenicity_classification_of_variant` returns PS3 even for variants in no range, so the range, + # not the criterion, tells us whether evidence exists. No strength means evaluated and not met. + if containing_range is None or evidence_strength_name is None: + pathogenicity_classification = AcmgClassification.UNCERTAIN_SIGNIFICANCE + elif criterion.name.startswith("B"): + pathogenicity_classification = AcmgClassification.BENIGN + else: + pathogenicity_classification = AcmgClassification.PATHOGENIC + + return FlatAnnotation( + functional_classification=functional_only.functional_classification, + acmg_criterion=criterion.value, + acmg_evidence_strength=evidence_strength_name, + acmg_evidence_outcome_code=acmg_evidence_outcome_code(criterion.value, evidence_strength_name), + pathogenicity_classification=pathogenicity_classification.name, + calibration_urn=functional_only.calibration_urn, + calibration_title=functional_only.calibration_title, + research_use_only=functional_only.research_use_only, + ) diff --git a/src/mavedb/lib/csv/__init__.py b/src/mavedb/lib/csv/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/mavedb/lib/csv/annotations.py b/src/mavedb/lib/csv/annotations.py new file mode 100644 index 000000000..3469d6d64 --- /dev/null +++ b/src/mavedb/lib/csv/annotations.py @@ -0,0 +1,109 @@ +"""Resolving each row's calibration interpretations for a CSV export. + +Shared by both exports, and kept out of ``columns`` because filling these cells needs the database. +""" + +from typing import Optional, Sequence + +from sqlalchemy import select +from sqlalchemy.orm import Session, selectinload + +from mavedb.lib.annotation.flatten import FlatAnnotation, flatten_annotation +from mavedb.lib.csv.entries import calibration_viewer +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_calibration_functional_classification_variant_association import ( + score_calibration_functional_classification_variants_association_table, +) +from mavedb.models.variant import Variant + + +def calibrations_for_namespaces( + db: Session, + calibration_namespaces: dict[str, str], + viewer: Optional[ScoreCalibrationViewer] = None, +) -> dict[str, ScoreCalibration]: + """Load the calibrations named by the requested namespaces, keyed by namespace. + + Looked up by the URN the caller named, not by what a score set offers: the namespace *is* the request. + Which means this, not discovery, is the gate — naming a private calibration's URN directly must not + serve its interpretation, so the viewer is applied here too. + """ + if not calibration_namespaces: + return {} + + calibrations = db.scalars( + select(ScoreCalibration) + .where(ScoreCalibration.urn.in_(list(calibration_namespaces.values()))) + .options( + selectinload(ScoreCalibration.functional_classifications).selectinload( + ScoreCalibrationFunctionalClassification.acmg_classification + ) + ) + ).all() + + by_urn = {str(calibration.urn): calibration for calibration in calibration_viewer(viewer).visible(calibrations)} + return {namespace: by_urn[urn] for namespace, urn in calibration_namespaces.items() if urn in by_urn} + + +def containing_classification_ids(db: Session, variant_ids: Sequence[int]) -> dict[int, set[int]]: + """Map each variant to the score-classification ids whose range contains it. + + One query over the association table, replacing the ORM membership check in + ``mavedb.lib.annotation.classification`` that loads every variant of every range once per range per + row — the dominant cost of these exports at score-set scale. + """ + if not variant_ids: + return {} + + membership: dict[int, set[int]] = {variant_id: set() for variant_id in variant_ids} + rows = db.execute( + select( + score_calibration_functional_classification_variants_association_table.c.variant_id, + score_calibration_functional_classification_variants_association_table.c.functional_classification_id, + ).where(score_calibration_functional_classification_variants_association_table.c.variant_id.in_(variant_ids)) + ).all() + for variant_id, classification_id in rows: + membership[variant_id].add(classification_id) + + return membership + + +def annotations_for_rows( + db: Session, + variants: Sequence[Variant], + mappings: Sequence[Optional[MappedVariant]], + calibration_namespaces: dict[str, str], + viewer: Optional[ScoreCalibrationViewer] = None, +) -> Optional[list[dict[str, Optional[FlatAnnotation]]]]: + """Flatten every row's interpretation under each requested calibration namespace. + + A calibration from a different score set than the row leaves that namespace empty: a score from one + assay carries no meaning under another's thresholds. So does one the caller may not read. + + Returns: + None when no calibration namespace was requested, so the caller can skip the work entirely. + """ + if not calibration_namespaces: + return None + + calibrations_by_ns = calibrations_for_namespaces(db, calibration_namespaces, viewer) + # TODO(#372): non-null id fields + membership = containing_classification_ids(db, [variant.id for variant in variants]) # type: ignore + + rows: list[dict[str, Optional[FlatAnnotation]]] = [] + for variant, mapping in zip(variants, mappings): + # TODO(#372): non-null id fields + contained = membership.get(variant.id, set()) # type: ignore + annotations: dict[str, Optional[FlatAnnotation]] = {} + for namespace in calibration_namespaces: + calibration = calibrations_by_ns.get(namespace) + if mapping is None or calibration is None or calibration.score_set_id != variant.score_set_id: + annotations[namespace] = None + else: + annotations[namespace] = flatten_annotation(mapping, calibration, contained) + rows.append(annotations) + + return rows diff --git a/src/mavedb/lib/csv/columns.py b/src/mavedb/lib/csv/columns.py new file mode 100644 index 000000000..7853a1d73 --- /dev/null +++ b/src/mavedb/lib/csv/columns.py @@ -0,0 +1,252 @@ +"""Column planning and row assembly for the CSV exports. + +Pure functions over already-fetched objects; nothing here touches the database. What a namespace *is* +lives in ``specs``; this module only applies it. +""" + +import csv +import io +from dataclasses import dataclass +from typing import Any, Iterable, Optional, Sequence + +from mavedb.lib.annotation.flatten import FlatAnnotation +from mavedb.lib.csv.namespaces import ( + CALIBRATION_NS_PATTERN, + CLINVAR_NS_PATTERN, + parse_calibration_namespace, + parse_clinvar_namespace, +) +from mavedb.lib.csv.specs import CORE_NAMESPACE, RowSource, namespace_spec +from mavedb.lib.mave.utils import NA_VALUE +from mavedb.lib.validation.utilities import is_null as validate_is_null +from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.variant import Variant + +_OUTPUT_NULL_STRINGS = frozenset({"none", "nan", "na", "undefined", "n/a", "null", "nil"}) + + +@dataclass(frozen=True) +class CsvColumnPlan: + namespaced_columns: dict[str, list[str]] + """Namespace -> list of column keys to emit for that namespace.""" + clinvar_namespaces: dict[str, str] + """Requested ClinVar namespace -> the ``"MM_YYYY"`` db_version it names.""" + calibration_namespaces: dict[str, str] + """Requested calibration namespace -> the calibration URN it names.""" + + +def _is_output_null(value: Any) -> bool: + """Whether *value* should be written as the NA sentinel rather than rendered. + + Distinct from ``lib.mave.utils.is_csv_null``, which decides whether a value read *from* an uploaded + file counts as missing: that one copes with pandas NA types and treats 0 specially. + """ + text = str(value).strip().lower() + return not text or text in _OUTPUT_NULL_STRINGS + + +def _value_or_na(value: Any, na_rep: str = NA_VALUE) -> str: + """Return the string representation of *value*, or *na_rep* if the value is null-ish.""" + if _is_output_null(value): + return na_rep + return str(value) + + +def _format_column_key(namespace: str, column_key: str, namespaced: bool = False) -> str: + """Shared key-formatting logic used by both header assembly and row assembly.""" + # Always namespaced regardless of the caller's preference: the release or calibration URN is what + # disambiguates columns when several are requested at once. + if CLINVAR_NS_PATTERN.match(namespace) or CALIBRATION_NS_PATTERN.match(namespace): + return f"{namespace}.{column_key}" + + if namespace == CORE_NAMESPACE: # core is never namespaced + return column_key + + if namespaced: + spec = namespace_spec(namespace) + prefix = spec.emit_under if spec is not None and spec.emit_under is not None else namespace + return f"{prefix}.{column_key}" + + return column_key + + +def plan_csv_columns(dataset_columns: dict, namespaces: list[str]) -> CsvColumnPlan: + """Build the namespaced column map and the ClinVar and calibration namespace mappings. + + An unknown namespace is kept with no columns rather than rejected — validating the vocabulary belongs + to the request layer. + """ + namespaced_columns: dict[str, list[str]] = {} + clinvar_namespaces: dict[str, str] = {} + calibration_namespaces: dict[str, str] = {} + + for namespace in dict.fromkeys([CORE_NAMESPACE, *namespaces]): + spec = namespace_spec(namespace) + namespaced_columns[namespace] = spec.columns(dataset_columns) if spec else [] + + db_version = parse_clinvar_namespace(namespace) + if db_version is not None: + clinvar_namespaces[namespace] = db_version + + calibration_urn = parse_calibration_namespace(namespace) + if calibration_urn is not None: + calibration_namespaces[namespace] = calibration_urn + + return CsvColumnPlan( + namespaced_columns=namespaced_columns, + clinvar_namespaces=clinvar_namespaces, + calibration_namespaces=calibration_namespaces, + ) + + +def assemble_csv_headers(namespaced_columns: dict[str, list[str]], namespaced: bool = False) -> list[str]: + """Build the flat column-header list from the namespace dict. + + Raises: + ValueError: if two namespaces resolve to the same header. Un-namespaced output strips the prefix + that would otherwise keep them apart, so requesting two namespaces that share a column name + would emit it twice; the callers that ask for un-namespaced output request one namespace each, + and this holds them to it rather than letting a future caller find out from a broken file. + """ + headers = [ + _format_column_key(namespace, col, namespaced) for namespace, cols in namespaced_columns.items() for col in cols + ] + + duplicates = sorted({header for header in headers if headers.count(header) > 1}) + if duplicates: + raise ValueError( + f"CSV namespaces resolve to duplicate columns: {', '.join(duplicates)}." + f" Requested namespaces: {', '.join(namespaced_columns)}." + ) + + return headers + + +def variant_to_csv_row( + variant: Variant, + columns: dict[str, list[str]], + mapping: Optional[MappedVariant] = None, + gnomad_data: Optional[GnomADVariant] = None, + clinvar_data_by_ns: Optional[dict[str, Optional[ClinicalControl]]] = None, + annotations_by_ns: Optional[dict[str, Optional[FlatAnnotation]]] = None, + match_type: Optional[str] = None, + namespaced: bool = False, + na_rep=NA_VALUE, +) -> dict[str, Any]: + """Format a variant into a dict containing the keys specified in *columns*. + + Args: + clinvar_data_by_ns, annotations_by_ns: per-row data for the parameterized namespaces, keyed by + requested namespace. A namespace with no entry renders as *na_rep*. + """ + row: dict[str, Any] = {} + + # Built once per row, not per namespace: a 100k-variant export with ten namespaces would otherwise + # build this dict, and walk `variant.data` twice, a million times over. + row_sources: dict[RowSource, Any] = { + RowSource.VARIANT: variant, + RowSource.MAPPING: mapping, + RowSource.GNOMAD: gnomad_data, + RowSource.MATCH_TYPE: match_type, + RowSource.SCORE_DATA: (variant.data or {}).get("score_data"), + RowSource.COUNT_DATA: (variant.data or {}).get("count_data"), + } + + for namespace, column_keys in columns.items(): + spec = namespace_spec(namespace) + if spec is None: + continue + + source: Any + # Only the parameterized namespaces carry a distinct datum per namespace. + if spec.source is RowSource.CLINVAR_ENTRY: + source = (clinvar_data_by_ns or {}).get(namespace) + elif spec.source is RowSource.ANNOTATION: + source = (annotations_by_ns or {}).get(namespace) + else: + source = row_sources[spec.source] + + for column_key in column_keys: + resolver = spec.resolver(column_key) + if resolver is None: + raise ValueError(f"unrecognized {namespace} column: {column_key}") + + row[_format_column_key(namespace, column_key, namespaced=namespaced)] = _value_or_na( + resolver(source), na_rep + ) + + return row + + +def variants_to_csv_rows( + variants: Sequence[Variant], + columns: dict[str, list[str]], + mappings: Optional[Sequence[Optional[MappedVariant]]] = None, + gnomad_data: Optional[Sequence[Optional[GnomADVariant]]] = None, + clinvar_data_by_ns: Optional[Sequence[Optional[dict[str, Optional[ClinicalControl]]]]] = None, + annotations_by_ns: Optional[Sequence[Optional[dict[str, Optional[FlatAnnotation]]]]] = None, + match_types: Optional[Sequence[Optional[str]]] = None, + namespaced: bool = False, + na_rep=NA_VALUE, +) -> Iterable[dict[str, Any]]: + """Format each variant into a dictionary row containing the keys specified in *columns*.""" + n = len(variants) + _mappings: Sequence[Optional[MappedVariant]] = mappings if mappings is not None else [None] * n + _gnomad: Sequence[Optional[GnomADVariant]] = gnomad_data if gnomad_data is not None else [None] * n + _clinvar: Sequence[Optional[dict[str, Optional[ClinicalControl]]]] = ( + clinvar_data_by_ns if clinvar_data_by_ns is not None else [None] * n + ) + _annotations: Sequence[Optional[dict[str, Optional[FlatAnnotation]]]] = ( + annotations_by_ns if annotations_by_ns is not None else [None] * n + ) + _match_types: Sequence[Optional[str]] = match_types if match_types is not None else [None] * n + return map( + lambda t: variant_to_csv_row( + t[0], + columns, + mapping=t[1], + gnomad_data=t[2], + clinvar_data_by_ns=t[3], + annotations_by_ns=t[4], + match_type=t[5], + namespaced=namespaced, + na_rep=na_rep, + ), + zip(variants, _mappings, _gnomad, _clinvar, _annotations, _match_types), + ) + + +def rows_to_csv(rows: Iterable[dict[str, Any]], columns: list[str]) -> str: + """Serialize *rows* to a CSV string headed by *columns*.""" + stream = io.StringIO() + writer = csv.DictWriter(stream, fieldnames=columns, quoting=csv.QUOTE_MINIMAL) + writer.writeheader() + writer.writerows(rows) + return stream.getvalue() + + +def drop_unused_hgvs_columns( + rows_data: Iterable[dict[str, Any]], columns: list[str] +) -> tuple[list[dict[str, Any]], list[str]]: + """Omit the HGVS coordinate columns this score set does not use. + + A protein-only score set never has ``hgvs_nt``; that is a property of the score set, not sparse data. + Limited to the three core HGVS columns on purpose — dropping data-dependent columns elsewhere would + make a download's shape vary with its contents. + + Assumes the "core" namespace is present, which ``plan_csv_columns`` guarantees. + """ + rows_data = list(rows_data) + columns_to_check = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + columns_to_remove = [] + + for col in columns_to_check: + if all(validate_is_null(row[col]) for row in rows_data): + columns_to_remove.append(col) + for row in rows_data: + row.pop(col, None) + + columns = [col for col in columns if col not in columns_to_remove] + return rows_data, columns diff --git a/src/mavedb/lib/csv/deprecated_params.py b/src/mavedb/lib/csv/deprecated_params.py new file mode 100644 index 000000000..3199fce5d --- /dev/null +++ b/src/mavedb/lib/csv/deprecated_params.py @@ -0,0 +1,106 @@ +"""Deprecated query parameters on the score-set CSV endpoints, kept working for backwards compatibility. + +``drop_na_columns`` and ``include_post_mapped_hgvs`` were renamed when the CSV export moved to a namespace +vocabulary. FastAPI ignores unknown query parameters, so a client still sending the old names would have +silently received different output rather than an error, and Galaxy calls these endpoints. + +Requests using a deprecated name get ``Deprecation`` and ``Warning`` response headers, the parameter is +marked deprecated in OpenAPI, and each use is logged so we can see who is left before removal. + +TODO(#XXX): remove this module once clients have migrated. +""" + +import logging +from dataclasses import dataclass, field +from typing import List, Optional + +from mavedb.lib.csv.namespaces import CsvNamespace +from mavedb.lib.logging.context import save_to_logging_context + +logger = logging.getLogger(__name__) + + +DROP_NA_COLUMNS_DESCRIPTION = ( + "Deprecated: use `drop_unused_hgvs_columns`, which names what it actually does. This parameter only" + " ever dropped the HGVS coordinate columns a score set does not use, never every NA column. It will" + " be removed in a future release; `drop_unused_hgvs_columns` wins if both are given." +) + +INCLUDE_POST_MAPPED_HGVS_DESCRIPTION = ( + "Deprecated: request the `mavedb` namespace instead, e.g. `?namespaces=scores&namespaces=mavedb`." + " Passing true here is equivalent to appending that namespace. It will be removed in a future release." +) + +INCLUDE_CUSTOM_COLUMNS_DESCRIPTION = ( + "Deprecated: request the `scores_custom` namespace instead. Passing true here is equivalent to" + " appending that namespace, whose columns are emitted under the `scores` prefix as before. It will be" + " removed in a future release." +) + + +@dataclass +class ResolvedCsvParams: + """The parameters an endpoint should act on, plus the headers telling the client what it sent.""" + + namespaces: List[str] + drop_unused_hgvs_columns: Optional[bool] + deprecations: dict[str, str] = field(default_factory=dict) + + def _record(self, name: str, replacement: str) -> None: + self.deprecations[name] = replacement + save_to_logging_context({"deprecated_query_parameters": sorted(self.deprecations), "deprecation_marker": True}) + logger.warning( + msg=f"Request used the deprecated query parameter '{name}'; it will be removed in a future" + f" release. Use '{replacement}' instead.", + extra={"deprecated_query_parameter": name, "replacement_query_parameter": replacement}, + ) + + @property + def response_headers(self) -> dict[str, str]: + """Headers announcing the deprecation to the client, or nothing at all for a current request.""" + if not self.deprecations: + return {} + + warnings = "; ".join( + f"{name} is deprecated, use {replacement}" for name, replacement in sorted(self.deprecations.items()) + ) + return { + # RFC 8594. No Sunset header: the removal release is not scheduled. + "Deprecation": "true", + "Warning": f'299 - "{warnings}"', + } + + +def resolve_deprecated_csv_params( + *, + namespaces: Optional[List[str]] = None, + drop_unused_hgvs_columns: Optional[bool] = None, + drop_na_columns: Optional[bool] = None, + include_post_mapped_hgvs: Optional[bool] = None, + include_custom_columns: Optional[bool] = None, +) -> ResolvedCsvParams: + """Fold the deprecated spellings into the current ones. + + The current name wins when both are given. The two boolean flags append a namespace rather than + replacing the requested ones, since both were always additive to whatever columns were asked for. + """ + resolved = ResolvedCsvParams( + namespaces=list(namespaces or []), + drop_unused_hgvs_columns=drop_unused_hgvs_columns, + ) + + if drop_unused_hgvs_columns is None and drop_na_columns is not None: + resolved.drop_unused_hgvs_columns = drop_na_columns + resolved._record("drop_na_columns", "drop_unused_hgvs_columns") + + if include_post_mapped_hgvs: + resolved._record("include_post_mapped_hgvs", "namespaces=mavedb") + if CsvNamespace.REFERENCE_HGVS not in resolved.namespaces: + resolved.namespaces.append(CsvNamespace.REFERENCE_HGVS) + + if include_custom_columns: + resolved._record("include_custom_columns", "namespaces=scores_custom") + if CsvNamespace.SCORES_CUSTOM not in resolved.namespaces: + resolved.namespaces.append(CsvNamespace.SCORES_CUSTOM) + + return resolved diff --git a/src/mavedb/lib/csv/entries.py b/src/mavedb/lib/csv/entries.py new file mode 100644 index 000000000..ba38b7f60 --- /dev/null +++ b/src/mavedb/lib/csv/entries.py @@ -0,0 +1,190 @@ +"""Shared pieces for advertising CSV columns: the entry a picker renders, the label builders, and the +two questions about a score set that decide whether a namespace is offerable. + +Each export owns its own discovery function, since what counts as "available" differs: the score-set CSV +asks about one score set, the variant CSV widens across every score set measuring the same allele. +""" + +from dataclasses import dataclass +from typing import Iterable, Optional, Sequence + +from sqlalchemy import and_, select +from sqlalchemy.orm import Session + +from mavedb.lib.annotation.util import score_calibration_may_be_used_for_annotation +from mavedb.lib.csv.namespaces import ( + CLINVAR_DB_NAME, + STATIC_CSV_NAMESPACE_LABELS, + CsvNamespaceGroup, + calibration_namespace_for_urn, + clinvar_namespace_for_db_version, + clinvar_namespace_label, + clinvar_namespace_sort_key, +) +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer +from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant + + +@dataclass(frozen=True) +class AvailableCsvNamespaceEntry: + """A namespace a record has data for, labeled and grouped for a picker.""" + + namespace: str + label: str + group: CsvNamespaceGroup + + score_set: Optional[ScoreSet] = None + """Owning score set, set for calibration namespaces only. + + A calibration means nothing against another score set's scores, and the variant CSV widens across + several score sets, so a picker needs this to tell their calibrations apart. + """ + + selected_by_default: bool = True + """Whether a picker should open with this group checked. + + False for research-use-only calibrations and for calibrations with no ranges. Answers only "what + should a dialog open on" . Do not read this as a publish/include policy. + """ + + research_use_only: bool = False + """Whether the data comes from a research-use-only calibration. + + Separate from ``selected_by_default`` so a consumer deciding what may be published can ask directly. + """ + + +def static_namespace_entry(namespace: str) -> AvailableCsvNamespaceEntry: + """Build the labeled entry for a static namespace.""" + label, group = STATIC_CSV_NAMESPACE_LABELS[namespace] + return AvailableCsvNamespaceEntry(namespace=namespace, label=label, group=group) + + +def clinvar_namespace_entries(namespaces: Iterable[str]) -> list[AvailableCsvNamespaceEntry]: + """Build labeled entries for ClinVar release namespaces, newest first, newest selected by default.""" + entries: list[AvailableCsvNamespaceEntry] = [] + # Chronological key, not string order: this sort decides which release opens checked. + for namespace in sorted(set(namespaces), key=clinvar_namespace_sort_key, reverse=True): + label = clinvar_namespace_label(namespace) + if label is not None: + entries.append( + AvailableCsvNamespaceEntry( + namespace=namespace, + label=label, + group=CsvNamespaceGroup.ANNOTATION, + selected_by_default=not entries, # first to survive labelling wins the default + ) + ) + + return entries + + +def calibration_viewer(viewer: Optional[ScoreCalibrationViewer]) -> ScoreCalibrationViewer: + """Resolve an omitted viewer to the anonymous one. + + A calibration carries its own ``private`` flag, and its READ permission is stricter than its score + set's: a private one is readable only by its owner, by contributors when it is investigator-provided, + or by an admin. Reading the score set is not enough, so every CSV path that names a calibration has to + ask separately. + + This is the single place the CSV package decides what an absent viewer means, and it means the public + subset: a call site that forgets to thread one serves what anyone could already see rather than + everything. The rule itself lives in ``ScoreCalibrationViewer``, so it is never restated here. + """ + return viewer if viewer is not None else ScoreCalibrationViewer() + + +def calibration_can_annotate(calibration: ScoreCalibration) -> bool: + """Whether a calibration can support either kind of annotation, and so fill any of its columns. + + False for a calibration with no score ranges, whose every cell would be NA. Research-use-only standing + is excluded from this question — it asks what a calibration *could* say, while who may see it is + ``ScoreCalibrationViewer``'s job. + """ + return any( + score_calibration_may_be_used_for_annotation( + calibration, + annotation_type=annotation_type, # type: ignore[arg-type] + allow_research_use_only_calibrations=True, + ) + for annotation_type in ("functional", "pathogenicity") + ) + + +def calibration_namespace_entries(calibrations: Iterable[ScoreCalibration]) -> list[AvailableCsvNamespaceEntry]: + """Build labeled entries for calibrations, named by title so a picker can identify them. + + Research-use-only calibrations (labelled with a prefix) and rangeless ones are offered but excluded + from the default selection. + """ + entries = [] + for calibration in sorted(calibrations, key=lambda c: (str(c.title or ""), str(c.urn or ""))): + if not calibration.urn: + continue + + title = str(calibration.title) if calibration.title else str(calibration.urn) + research_use_only = bool(calibration.research_use_only) + entries.append( + AvailableCsvNamespaceEntry( + namespace=calibration_namespace_for_urn(str(calibration.urn)), + label=f"Research Use Only: {title}" if research_use_only else title, + group=CsvNamespaceGroup.CALIBRATION, + score_set=calibration.score_set, + research_use_only=research_use_only, + selected_by_default=not research_use_only and calibration_can_annotate(calibration), + ) + ) + + return entries + + +def score_sets_have_current_mappings(db: Session, score_set_ids: Sequence[int]) -> bool: + """Whether any variant in these score sets has a current mapping. + + Gates the mapping-derived namespaces: any mapping in a score set means the variant CSV + should offer the namespaces, even if the variant in question is unmapped. + """ + if not score_set_ids: + return False + + return ( + db.scalars( + select(MappedVariant.id) + .join(MappedVariant.variant) + .where(and_(Variant.score_set_id.in_(score_set_ids), MappedVariant.current.is_(True))) + .limit(1) + ).first() + is not None + ) + + +def clinvar_release_namespaces(db: Session, score_set_ids: Sequence[int]) -> list[str]: + """Every ClinVar release namespace these score sets have data for. + + Scoped to the score set, not the measurement, so a variant with no record still gets NA columns — + an omitted column would read as "never consulted". Keyed on score set ids because deriving them + inside the query measured slower. + """ + if not score_set_ids: + return [] + + db_versions = db.scalars( + select(ClinicalControl.db_version) + .join(ClinicalControl.mapped_variants.of_type(MappedVariant)) + .join(MappedVariant.variant) + .where( + and_( + Variant.score_set_id.in_(score_set_ids), + MappedVariant.current.is_(True), + ClinicalControl.db_name == CLINVAR_DB_NAME, + ) + ) + .distinct() + ).all() + + namespaces = [clinvar_namespace_for_db_version(str(version)) for version in db_versions] + return [namespace for namespace in namespaces if namespace is not None] diff --git a/src/mavedb/lib/csv/fetch.py b/src/mavedb/lib/csv/fetch.py new file mode 100644 index 000000000..9244db15c --- /dev/null +++ b/src/mavedb/lib/csv/fetch.py @@ -0,0 +1,176 @@ +"""Fetching the rows a CSV export renders — a whole score set, or an explicit set of variants. + +Which relationships are eager-loaded follows from the requested namespaces, so a caller cannot forget one +and silently pay for an N+1. +""" + +from dataclasses import dataclass +from typing import Any, Optional, Sequence + +from sqlalchemy import Integer, and_, cast, func, select +from sqlalchemy.orm import Session, aliased, selectinload + +from mavedb.lib.csv.namespaces import CLINVAR_DB_NAME +from mavedb.lib.csv.specs import namespace_spec +from mavedb.lib.gnomad import GNOMAD_DATA_VERSION, GNOMAD_DB_NAME +from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant + + +@dataclass +class CsvFetchResult: + variants: list[Variant] + mappings: Optional[list[Optional[MappedVariant]]] + gnomad_data: Optional[list[Optional[GnomADVariant]]] + clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] + + +def fetch_variant_csv_data( + db: Session, + namespaced_columns: dict[str, list[str]], + clinvar_namespaces: dict[str, str], + *, + score_set: Optional[ScoreSet] = None, + variant_ids: Optional[Sequence[int]] = None, + mapped_variant_ids: Optional[Sequence[int]] = None, + start: Optional[int] = None, + limit: Optional[int] = None, +) -> CsvFetchResult: + """Fetch variant data from the database for CSV generation. + + Args: + score_set: every variant in a score set, ordered by URN suffix. Mutually exclusive with + *variant_ids*, which returns an explicit set in the order given. Exactly one is required. + mapped_variant_ids: pins which mapping stands for each variant. Required from a caller that has + already chosen one, since re-resolving on ``current`` alone could pick a different row and + emit a variant twice — nothing in the schema stops two mappings claiming to be current. + """ + if (score_set is None) == (variant_ids is None): + raise ValueError("exactly one of score_set or variant_ids must be provided") + + # Driven by the namespaces' own descriptors, so a namespace cannot declare a relationship-backed + # column and then quietly not have it loaded. + specs = [spec for spec in (namespace_spec(ns) for ns in namespaced_columns) if spec is not None] + + need_mappings = any(spec.needs_mappings for spec in specs) + need_gnomad = any(spec.needs_gnomad for spec in specs) + need_score_set = any(spec.needs_score_set for spec in specs) + + variants: list[Variant] = [] + mappings: Optional[list[Optional[MappedVariant]]] = [] if need_mappings else None + gnomad_data_list: Optional[list[Optional[GnomADVariant]]] = [] if need_gnomad else None + + select_columns: list[Any] = [Variant] + if need_mappings: + select_columns.append(MappedVariant) + if need_gnomad: + select_columns.append(GnomADVariant) + + query = select(*select_columns) + + if score_set is not None: + query = query.where(Variant.score_set_id == score_set.id).order_by( + cast(func.split_part(Variant.urn, "#", 2), Integer) + ) + else: + query = query.where(Variant.id.in_(variant_ids or [])) + + if need_score_set: + query = query.options( + selectinload(Variant.score_set).selectinload(ScoreSet.score_calibrations), + selectinload(Variant.score_set).selectinload(ScoreSet.target_genes), + ) + + if need_mappings: + mapping_on_clause = ( + and_(Variant.id == MappedVariant.variant_id, MappedVariant.id.in_(mapped_variant_ids)) + if mapped_variant_ids is not None + else and_(Variant.id == MappedVariant.variant_id, MappedVariant.current.is_(True)) + ) + query = query.join(MappedVariant, mapping_on_clause, isouter=True) + + # Version predicate belongs in the ON clause: in a WHERE it would drop any variant linked only to + # other-version gnomAD records from the CSV entirely, instead of reporting its frequency as NA. + if need_gnomad: + query = query.join( + MappedVariant.gnomad_variants.of_type(GnomADVariant).and_( + GnomADVariant.db_name == GNOMAD_DB_NAME, GnomADVariant.db_version == GNOMAD_DATA_VERSION + ), + isouter=True, + ) + + if start: + query = query.offset(start) + if limit: + query = query.limit(limit) + + result = db.execute(query).all() + + # Postgres does not preserve IN-list order, so restore the caller's ordering. + if variant_ids is not None: + position = {variant_id: index for index, variant_id in enumerate(variant_ids)} + result = sorted(result, key=lambda row: position.get(row[0].id, len(position))) + + for row in result: + variant = row[0] + variants.append(variant) + + if need_mappings and mappings is not None: + mappings.append(row[1]) + + if need_gnomad and gnomad_data_list is not None: + idx = 2 if need_mappings else 1 + gnomad_data_list.append(row[idx]) + + clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] = None + if clinvar_namespaces and mappings is not None: + mv_ids = [m.id for m in mappings if m is not None] + + # One query per namespace, since each names a different release; keyed by MappedVariant id and + # projected back onto row order below. + clinvar_data_map: dict[str, dict[int, Optional[ClinicalControl]]] = {} + for ns, db_version in clinvar_namespaces.items(): + mv_to_cc: dict[int, Optional[ClinicalControl]] = {} + if mv_ids: + aliased_cc = aliased(ClinicalControl) + cc_query = ( + select( + mapped_variants_clinical_controls_association_table.c.mapped_variant_id, + aliased_cc, + ) + .join( + aliased_cc, + mapped_variants_clinical_controls_association_table.c.clinical_control_id == aliased_cc.id, + ) + .where( + and_( + mapped_variants_clinical_controls_association_table.c.mapped_variant_id.in_(mv_ids), + aliased_cc.db_name == CLINVAR_DB_NAME, + aliased_cc.db_version == db_version, + ) + ) + ) + + for mv_id, cc in db.execute(cc_query).all(): + mv_to_cc[mv_id] = cc + + clinvar_data_map[ns] = mv_to_cc + + clinvar_per_variant = [ + { + ns: mv_to_cc.get(mapping.id) if mapping is not None and mapping.id is not None else None + for ns, mv_to_cc in clinvar_data_map.items() + } + for mapping in mappings + ] + + return CsvFetchResult( + variants=variants, + mappings=mappings, + gnomad_data=gnomad_data_list, + clinvar_per_variant=clinvar_per_variant, + ) diff --git a/src/mavedb/lib/csv/namespaces.py b/src/mavedb/lib/csv/namespaces.py new file mode 100644 index 000000000..0565c6a8c --- /dev/null +++ b/src/mavedb/lib/csv/namespaces.py @@ -0,0 +1,254 @@ +"""The vocabulary of CSV column namespaces: names, labels, validation. + +Most namespaces are fixed names. Two families are parameterized, since their columns depend on which +record the caller wants: ``clinvar.YYYY_MM`` and ``calibration.``. A parameterized namespace carries +its parameter into the column header, keeping values traceable without a separate provenance column. +""" + +import re +from enum import StrEnum +from typing import Annotated, Optional + +from pydantic import AfterValidator, WithJsonSchema + +from mavedb.lib.validation.urn_re import MAVEDB_CALIBRATION_URN_PATTERN + + +class CsvNamespaceGroup(StrEnum): + """Presentational grouping, so a client can section a namespace picker.""" + + DATA = "data" + ANNOTATION = "annotation" + CALIBRATION = "calibration" + PROVENANCE = "provenance" + + +class CsvNamespace(StrEnum): + """Namespaces whose column sets are fixed and take no parameter. + + The parameterized families cannot be members, so requests are validated against this enum *and* those + patterns. See ``is_valid_csv_namespace``. + """ + + SCORES = "scores" + """The one score column every score set is required to define.""" + + SCORES_CUSTOM = "scores_custom" + """The remaining score columns the investigator uploaded. + + A request token only: its columns are emitted under the ``scores`` prefix, since they are score + columns. Splitting selection from emission is what let this replace the ``include_custom_columns`` + flag without changing a published header. + """ + + COUNTS = "counts" + + # Value frozen as "mavedb", and its columns keep their post_mapped_* names: both are published, and + # the score-set histogram parses mavedb.post_mapped_hgvs_c by name. + REFERENCE_HGVS = "mavedb" + + VEP = "vep" + GNOMAD = "gnomad" + CLINGEN = "clingen" + SCORE_SET = "score_set" + RELATIONSHIP = "relationship" + + +STATIC_CSV_NAMESPACES: tuple[str, ...] = tuple(ns.value for ns in CsvNamespace) +"""The static namespace values, in declaration order, for iteration and documentation.""" + +STATIC_CSV_NAMESPACE_LABELS: dict[str, tuple[str, CsvNamespaceGroup]] = { + CsvNamespace.SCORES: ("Score", CsvNamespaceGroup.DATA), + CsvNamespace.SCORES_CUSTOM: ("Investigator-provided score columns", CsvNamespaceGroup.DATA), + CsvNamespace.COUNTS: ("Counts", CsvNamespaceGroup.DATA), + CsvNamespace.CLINGEN: ("ClinGen allele ID", CsvNamespaceGroup.ANNOTATION), + CsvNamespace.REFERENCE_HGVS: ("Reference-frame HGVS", CsvNamespaceGroup.ANNOTATION), + CsvNamespace.VEP: ("VEP consequence", CsvNamespaceGroup.ANNOTATION), + CsvNamespace.GNOMAD: ("gnomAD allele frequency", CsvNamespaceGroup.ANNOTATION), + CsvNamespace.SCORE_SET: ("Score set and target gene", CsvNamespaceGroup.PROVENANCE), + CsvNamespace.RELATIONSHIP: ("Relationship to the requested variant", CsvNamespaceGroup.PROVENANCE), +} +"""Label and group for each static namespace, so a client need not maintain its own mapping. + +The parameterized families are labeled from their parameter instead; see ``clinvar_namespace_label``. +""" + + +_CLINVAR_MONTH_NAMES = ( + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", +) + + +CLINVAR_NS_PATTERN = re.compile(r"^clinvar\.(\d+)_(0[1-9]|1[0-2])$") +"""Pattern for ClinVar namespaces of the form ``"clinvar.YEAR_MONTH"``, e.g. ``clinvar.2024_01``.""" + + +CLINVAR_DB_NAME = "ClinVar" +"""The ``clinical_controls.db_name`` a ``clinvar.*`` namespace selects on.""" + + +def parse_clinvar_namespace(ns: str) -> Optional[str]: + """Parse a ClinVar namespace into the ``db_version`` stored in ``clinical_controls``. + + Namespaces are of the form ``"clinvar.YEAR_MONTH"`` (e.g. ``"clinvar.2024_01"`` for January 2024). + The corresponding ``db_version`` is ``"MONTH_YEAR"`` (e.g. ``"01_2024"``). + + Returns ``None`` if *ns* does not match the expected pattern. + """ + m = CLINVAR_NS_PATTERN.match(ns) + if not m: + return None + year, month = m.group(1), m.group(2) + return f"{month}_{year}" + + +def parse_clinvar_db_version(db_version: str) -> Optional[tuple[int, int]]: + """Parse a ClinVar ``"MM_YYYY"`` db_version into ``(year, month)``. + + Returns ``None`` when *db_version* is not in the expected form. The tuple orders chronologically, so + it doubles as a sort key for picking the most recent release. + """ + try: + month, year = db_version.split("_") + return (int(year), int(month)) + except (ValueError, AttributeError): + return None + + +_UNDATED_CLINVAR_SORT_KEY = (-1, -1) +"""Sorts before every real release, so a namespace we cannot date never wins a "newest" comparison.""" + + +def clinvar_namespace_sort_key(ns: str) -> tuple[int, int]: + """Chronological sort key for a ClinVar release namespace. + + Use this for every "which release is newest" decision rather than comparing namespace strings: the + year group is unpadded, so ``"clinvar.999_12" > "clinvar.2025_01"`` lexically. Non-release namespaces + sort before every real one. + """ + match = CLINVAR_NS_PATTERN.match(ns) + if not match: + return _UNDATED_CLINVAR_SORT_KEY + return (int(match.group(1)), int(match.group(2))) + + +CALIBRATION_NS_PATTERN = re.compile(rf"^calibration\.({MAVEDB_CALIBRATION_URN_PATTERN})$") +"""Pattern for calibration namespaces of the form ``"calibration."``.""" + + +def parse_calibration_namespace(ns: str) -> Optional[str]: + """Parse a calibration namespace into the calibration URN it names. + + Returns ``None`` if *ns* does not match the expected pattern. + """ + match = CALIBRATION_NS_PATTERN.match(ns) + if not match: + return None + return match.group(1) + + +def calibration_namespace_for_urn(urn: str) -> str: + """Build the namespace naming a calibration URN.""" + return f"calibration.{urn}" + + +def clinvar_namespace_for_db_version(db_version: str) -> Optional[str]: + """Build the namespace naming a ClinVar release from its ``"MM_YYYY"`` db_version. + + Returns ``None`` when *db_version* is not in the expected form. + """ + parsed = parse_clinvar_db_version(db_version) + if parsed is None: + return None + year, month = parsed + return f"clinvar.{year}_{month:02d}" + + +def clinvar_namespace_label(ns: str) -> Optional[str]: + """Human-readable label for a ClinVar release namespace, e.g. ``"ClinVar significance (November 2024)"``. + + Returns ``None`` when *ns* is not a ClinVar namespace. + """ + match = CLINVAR_NS_PATTERN.match(ns) + if not match: + return None + year, month = int(match.group(1)), int(match.group(2)) + return f"ClinVar significance ({_CLINVAR_MONTH_NAMES[month - 1]} {year})" + + +_STATIC_CSV_NAMESPACE_VALUES = frozenset(STATIC_CSV_NAMESPACES) +"""Membership set. ``"scores" in CsvNamespace`` raises TypeError on Python 3.11, so test against this.""" + + +def is_valid_csv_namespace(ns: str) -> bool: + """Whether *ns* is a namespace any CSV endpoint will accept.""" + return ( + ns in _STATIC_CSV_NAMESPACE_VALUES + or CLINVAR_NS_PATTERN.match(ns) is not None + or CALIBRATION_NS_PATTERN.match(ns) is not None + ) + + +CSV_NAMESPACE_ERROR_MESSAGE = ( + "must be one of " + + ", ".join(f'"{ns}"' for ns in STATIC_CSV_NAMESPACES) + + ', a ClinVar release namespace of the form "clinvar.YEAR_MONTH" (e.g. "clinvar.2024_01"),' + ' or a calibration namespace of the form "calibration."' +) + + +def _validated_csv_namespace(ns: str) -> str: + """Pydantic validator backing ``CsvNamespaceStr``.""" + if not is_valid_csv_namespace(ns): + raise ValueError(CSV_NAMESPACE_ERROR_MESSAGE) + return ns + + +CsvNamespaceStr = Annotated[ + str, + AfterValidator(_validated_csv_namespace), + # Hand-declared because no Python type expresses "closed enum OR two open patterns". FastAPI then + # rejects bad values itself, so endpoints need no vocabulary check. + # + # Caveat: this does not reach clients as a *type*. openapi-typescript narrows an enum mixed with + # patterns to plain `string`, so generated clients see `string[]` and cannot check a namespace name at + # compile time. Only own-component schemas (CsvNamespaceGroup) survive as a union. + WithJsonSchema( + { + "type": "string", + "anyOf": [ + {"enum": list(STATIC_CSV_NAMESPACES)}, + {"pattern": CLINVAR_NS_PATTERN.pattern}, + {"pattern": CALIBRATION_NS_PATTERN.pattern}, + ], + } + ), +] +"""The type for a ``namespaces`` query-parameter element on any CSV endpoint. + +Use ``Optional[List[CsvNamespaceStr]]`` and FastAPI handles validation and documentation. +""" + + +CSV_NAMESPACES_PARAM_DESCRIPTION = ( + "One or more groups of columns to include. Naming any group replaces the default set rather than " + "adding to it, so list every group you want. Fixed groups: " + + ", ".join(f'"{ns}"' for ns in STATIC_CSV_NAMESPACES) + + '. Versioned groups: "clinvar.YEAR_MONTH" (e.g. "clinvar.2024_01") for one ClinVar release, and ' + '"calibration." for one score calibration\'s functional and ACMG interpretation. ' + "Several ClinVar and calibration namespaces may be requested at once; each carries its release or " + "URN in the column header. To discover which namespaces are available for a record, query the " + "`csv-namespaces` endpoint." +) +"""Shared OpenAPI description for the ``namespaces`` query parameter on every CSV endpoint.""" diff --git a/src/mavedb/lib/csv/score_set.py b/src/mavedb/lib/csv/score_set.py new file mode 100644 index 000000000..9aafad32d --- /dev/null +++ b/src/mavedb/lib/csv/score_set.py @@ -0,0 +1,123 @@ +"""The score-set CSV export: every variant in one score set, and the columns it can offer.""" + +from typing import List, Optional + +from sqlalchemy import and_, select +from sqlalchemy.orm import Session, selectinload + +from mavedb.lib.csv.annotations import annotations_for_rows +from mavedb.lib.csv.columns import ( + assemble_csv_headers, + drop_unused_hgvs_columns, + plan_csv_columns, + rows_to_csv, + variants_to_csv_rows, +) +from mavedb.lib.csv.entries import ( + AvailableCsvNamespaceEntry, + calibration_namespace_entries, + calibration_viewer, + clinvar_namespace_entries, + clinvar_release_namespaces, + score_sets_have_current_mappings, + static_namespace_entry, +) +from mavedb.lib.csv.fetch import fetch_variant_csv_data +from mavedb.lib.csv.namespaces import CsvNamespace +from mavedb.lib.mave.constants import REQUIRED_SCORE_COLUMN +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_set import ScoreSet + + +def get_score_set_variants_as_csv( + db: Session, + score_set: ScoreSet, + namespaces: List[str], + namespaced: bool = False, + start: Optional[int] = None, + limit: Optional[int] = None, + drop_unused_hgvs_columns_flag: Optional[bool] = None, + viewer: Optional[ScoreCalibrationViewer] = None, +) -> str: + """Get the variant data from a score set as a CSV string.""" + assert type(score_set.dataset_columns) is dict + + plan = plan_csv_columns(score_set.dataset_columns, namespaces) + + fetched = fetch_variant_csv_data( + db, + plan.namespaced_columns, + plan.clinvar_namespaces, + score_set=score_set, + start=start, + limit=limit, + ) + + mappings = fetched.mappings or [None] * len(fetched.variants) + rows_data = variants_to_csv_rows( + fetched.variants, + columns=plan.namespaced_columns, + namespaced=namespaced, + mappings=fetched.mappings, + gnomad_data=fetched.gnomad_data, + clinvar_data_by_ns=fetched.clinvar_per_variant, + annotations_by_ns=annotations_for_rows(db, fetched.variants, mappings, plan.calibration_namespaces, viewer), + ) + + rows_columns = assemble_csv_headers(plan.namespaced_columns, namespaced=namespaced) + + if drop_unused_hgvs_columns_flag: + rows_data, rows_columns = drop_unused_hgvs_columns(rows_data, rows_columns) + + return rows_to_csv(rows_data, rows_columns) + + +def available_score_set_csv_namespaces( + db: Session, + score_set: ScoreSet, + viewer: Optional[ScoreCalibrationViewer] = None, +) -> list[AvailableCsvNamespaceEntry]: + """Every namespace the score-set CSV can serve data for, labeled and grouped for a picker. + + Its own endpoint rather than a field on the score-set response: it costs several queries and is only + needed when a download dialog opens. A namespace absent here is still accepted by the CSV endpoint; + it just produces a column of NA. + """ + dataset_columns = score_set.dataset_columns if isinstance(score_set.dataset_columns, dict) else {} + # TODO(#372): non-null id fields + score_set_ids: list[int] = [score_set.id] # type: ignore + + score_columns = [str(column) for column in dataset_columns.get("score_columns", [])] + + entries: list[AvailableCsvNamespaceEntry] = [] + if score_columns: + entries.append(static_namespace_entry(CsvNamespace.SCORES)) + if any(column != REQUIRED_SCORE_COLUMN for column in score_columns): + entries.append(static_namespace_entry(CsvNamespace.SCORES_CUSTOM)) + if dataset_columns.get("count_columns"): + entries.append(static_namespace_entry(CsvNamespace.COUNTS)) + + entries.append(static_namespace_entry(CsvNamespace.SCORE_SET)) # always its own provenance + + if score_sets_have_current_mappings(db, score_set_ids): + entries.extend( + static_namespace_entry(ns) + for ns in (CsvNamespace.REFERENCE_HGVS, CsvNamespace.VEP, CsvNamespace.GNOMAD, CsvNamespace.CLINGEN) + ) + entries.extend(clinvar_namespace_entries(clinvar_release_namespaces(db, score_set_ids))) + + # Every calibration the score set defines is offered, rangeless ones included. + calibrations = db.scalars( + select(ScoreCalibration) + .options( + selectinload(ScoreCalibration.score_set), + selectinload(ScoreCalibration.functional_classifications), # read by the eligibility check + ) + .where(and_(ScoreCalibration.score_set_id == score_set.id, ScoreCalibration.urn.is_not(None))) + ).all() + entries.extend(calibration_namespace_entries(calibration_viewer(viewer).visible(calibrations))) + + # `relationship` is absent by design: match_type describes a row's relation to a requested record, + # which only the variant CSV has. + return entries diff --git a/src/mavedb/lib/csv/specs.py b/src/mavedb/lib/csv/specs.py new file mode 100644 index 000000000..b3b671598 --- /dev/null +++ b/src/mavedb/lib/csv/specs.py @@ -0,0 +1,254 @@ +"""What each CSV namespace is: the columns it produces and how each is read off a row.""" + +from dataclasses import dataclass +from enum import StrEnum +from operator import attrgetter +from typing import Callable, Optional + +from mavedb.lib.csv.namespaces import CALIBRATION_NS_PATTERN, CLINVAR_NS_PATTERN, CsvNamespace +from mavedb.lib.mave.constants import REQUIRED_SCORE_COLUMN +from mavedb.lib.variants import get_digest_from_post_mapped, get_hgvs_from_post_mapped, is_hgvs_g, is_hgvs_p +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.variant import Variant + +# One entry per namespace, so adding one is a single edit. This previously took three unrelated changes +# — column plan, row builder, fetch-layer eager loading — with nothing to catch a partial addition. + + +CORE_NAMESPACE = "core" +"""The identity columns every export carries, never namespaced and never opted out of.""" + + +class DatasetColumnSelection(StrEnum): + """Which of a ``dataset_columns`` entry's columns a namespace claims. + + Only the score columns are split, because ``score`` is the one column dataframe validation mandates, + which makes "the required column" and "everything else" well defined. + """ + + ALL = "all" + + REQUIRED_SCORE_ONLY = "required_score_only" + """Just ``score``, without consulting the record: dataframe validation mandates the column, so callers + with no ``dataset_columns`` to hand (the variant CSV) still resolve it.""" + + EXCEPT_REQUIRED_SCORE = "except_required_score" + """Everything else, which only the record can enumerate.""" + + +class RowSource(StrEnum): + """Which per-row datum a namespace's resolvers are called with.""" + + VARIANT = "variant" + MAPPING = "mapping" + GNOMAD = "gnomad" + MATCH_TYPE = "match_type" + SCORE_DATA = "score_data" + COUNT_DATA = "count_data" + + # Parameterized namespaces are keyed by the namespace string, since one row carries a separate datum + # for every requested release or calibration. + CLINVAR_ENTRY = "clinvar_entry" + ANNOTATION = "annotation" + + +@dataclass(frozen=True) +class CsvNamespaceSpec: + """Everything one namespace contributes to an export.""" + + source: RowSource + """Which per-row datum the resolvers are called with.""" + + resolvers: Optional[dict[str, Callable]] = None + """Column key -> how to read it off *source*. None means the columns are not known ahead of time and + are read by key, which is how a score set's own score and count columns work.""" + + dataset_columns_key: Optional[str] = None + """The ``dataset_columns`` entry listing this namespace's columns, for namespaces whose columns come + from the score set rather than from this module.""" + + dataset_columns: DatasetColumnSelection = DatasetColumnSelection.ALL + """Which of that entry's columns this namespace claims.""" + + emit_under: Optional[str] = None + """Prefix these columns are emitted under, when it differs from the namespace's own name. + + Lets a namespace be a request token without becoming a column prefix, so ``scores_custom`` selects + while its columns stay ``scores.*``. None means emit under this namespace's own name. + """ + + needs_mappings: bool = False + """Whether the fetch layer has to load the row's mapping for this namespace to work.""" + needs_gnomad: bool = False + """Whether the fetch layer has to load the row's gnomAD data for this namespace to work.""" + needs_score_set: bool = False + """Whether the fetch layer has to load the row's score set for this namespace to work.""" + + def columns(self, dataset_columns: dict) -> list[str]: + """The column keys this namespace produces for a given score set.""" + if self.resolvers is not None: + return list(self.resolvers.keys()) + if self.dataset_columns_key is None: + return [] + + if self.dataset_columns is DatasetColumnSelection.REQUIRED_SCORE_ONLY: + return [REQUIRED_SCORE_COLUMN] + + available = [str(column) for column in dataset_columns.get(self.dataset_columns_key, [])] + if self.dataset_columns is DatasetColumnSelection.EXCEPT_REQUIRED_SCORE: + return [column for column in available if column != REQUIRED_SCORE_COLUMN] + + return available + + def resolver(self, column_key: str) -> Optional[Callable]: + """How to read *column_key* off a row, or None if it is read by key rather than by resolver.""" + if self.resolvers is not None: + return self.resolvers.get(column_key) + + # Dynamic columns are read straight out of the variant's score or count data by name. + return _optional(lambda data: data.get(column_key)) + + +def _post_mapped_hgvs_g(mapping: Optional[MappedVariant]) -> Optional[str]: + """The genomic HGVS expression, falling back to one parsed out of the post-mapped VRS object.""" + if mapping is None: + return None + if mapping.hgvs_g: + return str(mapping.hgvs_g) + fallback = get_hgvs_from_post_mapped(mapping.post_mapped) if mapping.post_mapped else None + return fallback if fallback is not None and is_hgvs_g(fallback) else None + + +def _post_mapped_hgvs_p(mapping: Optional[MappedVariant]) -> Optional[str]: + """The protein HGVS expression, falling back to one parsed out of the post-mapped VRS object.""" + if mapping is None: + return None + if mapping.hgvs_p: + return str(mapping.hgvs_p) + fallback = get_hgvs_from_post_mapped(mapping.post_mapped) if mapping.post_mapped else None + return fallback if fallback is not None and is_hgvs_p(fallback) else None + + +def _post_mapped_vrs_digest(mapping: Optional[MappedVariant]) -> Optional[str]: + """The digest of the post-mapped VRS object, or None if there is no post-mapped object.""" + if mapping is None or not mapping.post_mapped: + return None + return get_digest_from_post_mapped(mapping.post_mapped) + + +def _target_genes(variant: Variant) -> Optional[str]: + """The target genes of a variant's score set, joined by ``"; "`` or None if there are none.""" + if not variant.score_set: + return None + return "; ".join(str(tg.name) for tg in variant.score_set.target_genes if tg.name) or None + + +def _optional(getter: Callable) -> Callable: + """Lift a resolver over a source that may be absent, which is how a row reports "no data here".""" + return lambda source: getter(source) if source is not None else None + + +_NAMESPACE_SPECS: dict[str, CsvNamespaceSpec] = { + CORE_NAMESPACE: CsvNamespaceSpec( + source=RowSource.VARIANT, + resolvers={ + "accession": attrgetter("urn"), + "hgvs_nt": attrgetter("hgvs_nt"), + "hgvs_splice": attrgetter("hgvs_splice"), + "hgvs_pro": attrgetter("hgvs_pro"), + }, + ), + CsvNamespace.SCORES: CsvNamespaceSpec( + source=RowSource.SCORE_DATA, + dataset_columns_key="score_columns", + dataset_columns=DatasetColumnSelection.REQUIRED_SCORE_ONLY, + ), + CsvNamespace.SCORES_CUSTOM: CsvNamespaceSpec( + source=RowSource.SCORE_DATA, + dataset_columns_key="score_columns", + dataset_columns=DatasetColumnSelection.EXCEPT_REQUIRED_SCORE, + emit_under=CsvNamespace.SCORES, + ), + CsvNamespace.COUNTS: CsvNamespaceSpec(source=RowSource.COUNT_DATA, dataset_columns_key="count_columns"), + # TODO(#784): under the allele-centric (RT) substrate these move off MappedVariant onto the Allele. + # Both are reached through `mapping`, so each becomes a one-line hop, not a column-contract change. + CsvNamespace.REFERENCE_HGVS: CsvNamespaceSpec( + source=RowSource.MAPPING, + resolvers={ + "post_mapped_hgvs_g": _post_mapped_hgvs_g, + "post_mapped_hgvs_p": _post_mapped_hgvs_p, + "post_mapped_hgvs_c": _optional(lambda mapping: mapping.hgvs_c), + "post_mapped_hgvs_at_assay_level": _optional(lambda mapping: mapping.hgvs_assay_level), + "post_mapped_vrs_digest": _post_mapped_vrs_digest, + }, + needs_mappings=True, + ), + CsvNamespace.VEP: CsvNamespaceSpec( + source=RowSource.MAPPING, + resolvers={"vep_functional_consequence": _optional(lambda mapping: mapping.vep_functional_consequence)}, + needs_mappings=True, + ), + CsvNamespace.GNOMAD: CsvNamespaceSpec( + source=RowSource.GNOMAD, + resolvers={"gnomad_af": _optional(lambda gnomad: gnomad.allele_frequency)}, + needs_mappings=True, + needs_gnomad=True, + ), + CsvNamespace.CLINGEN: CsvNamespaceSpec( + source=RowSource.MAPPING, + resolvers={"clingen_allele_id": _optional(lambda mapping: mapping.clingen_allele_id)}, + needs_mappings=True, + ), + CsvNamespace.SCORE_SET: CsvNamespaceSpec( + source=RowSource.VARIANT, + resolvers={ + "score_set_urn": lambda variant: variant.score_set.urn if variant.score_set else None, + "target_gene": _target_genes, + }, + needs_score_set=True, + ), + # TODO(#784): once the variant CSV emits sibling rows, report the shared `projection_group` here + # alongside `match_type`, so a consumer can tell a projected sibling from an independent equivalent. + CsvNamespace.RELATIONSHIP: CsvNamespaceSpec( + source=RowSource.MATCH_TYPE, + # Caller-supplied: only an export that widens beyond one record knows how a row relates to it. + resolvers={"match_type": lambda match_type: match_type}, + ), +} + + +_CLINVAR_SPEC = CsvNamespaceSpec( + source=RowSource.CLINVAR_ENTRY, + resolvers={ + "clinical_significance": _optional(attrgetter("clinical_significance")), + "clinical_review_status": _optional(attrgetter("clinical_review_status")), + }, + needs_mappings=True, +) + +_CALIBRATION_SPEC = CsvNamespaceSpec( + source=RowSource.ANNOTATION, + # The calibration's URN is carried in the column header, so it is not repeated as a column. + resolvers={ + "title": _optional(attrgetter("calibration_title")), + "research_use_only": _optional(attrgetter("research_use_only")), + "functional_classification": _optional(attrgetter("functional_classification")), + "acmg_criterion": _optional(attrgetter("acmg_criterion")), + "acmg_evidence_strength": _optional(attrgetter("acmg_evidence_strength")), + "acmg_evidence_outcome_code": _optional(attrgetter("acmg_evidence_outcome_code")), + "pathogenicity_classification": _optional(attrgetter("pathogenicity_classification")), + }, + needs_mappings=True, + needs_score_set=True, +) + + +def namespace_spec(namespace: str) -> Optional[CsvNamespaceSpec]: + """The descriptor for *namespace*, or None if it names nothing this module can produce.""" + if namespace in _NAMESPACE_SPECS: + return _NAMESPACE_SPECS[namespace] + if CLINVAR_NS_PATTERN.match(namespace): + return _CLINVAR_SPEC + if CALIBRATION_NS_PATTERN.match(namespace): + return _CALIBRATION_SPEC + return None diff --git a/src/mavedb/lib/csv/variant.py b/src/mavedb/lib/csv/variant.py new file mode 100644 index 000000000..61e657e76 --- /dev/null +++ b/src/mavedb/lib/csv/variant.py @@ -0,0 +1,385 @@ +"""Clinically-oriented, variant-level CSV export. + +Serves the same interpretation as the variant-level VA-Spec JSON download, but flat: a manuscript +reviewer found the ACMG evidence codes clinically inaccessible when buried in nested evidence lines. + +Column layout, fetching, NA handling, and serialization come from the shared engine in this package. +Only the variant-scoped parts live here: finding measurements that share a ClinGen allele, and choosing +default calibration and ClinVar namespaces. +""" + +import logging +from typing import Any, Callable, Optional + +from sqlalchemy import and_, select +from sqlalchemy.orm import Session, selectinload + +from mavedb.lib.csv.annotations import annotations_for_rows +from mavedb.lib.csv.columns import ( + assemble_csv_headers, + plan_csv_columns, + rows_to_csv, + variants_to_csv_rows, +) +from mavedb.lib.csv.entries import ( + AvailableCsvNamespaceEntry, + calibration_can_annotate, + calibration_namespace_entries, + clinvar_namespace_entries, + clinvar_release_namespaces, + score_sets_have_current_mappings, + static_namespace_entry, + calibration_viewer, +) +from mavedb.lib.csv.fetch import fetch_variant_csv_data +from mavedb.lib.csv.namespaces import ( + CsvNamespace, + calibration_namespace_for_urn, + clinvar_namespace_sort_key, +) +from mavedb.lib.mave.utils import NA_VALUE +from mavedb.lib.urns import score_set_urn_sort_key, variant_urn_sort_key +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant + +logger = logging.getLogger(__name__) + + +ALWAYS_AVAILABLE_NAMESPACES: list[str] = [ + CsvNamespace.SCORES, + CsvNamespace.SCORE_SET, + CsvNamespace.RELATIONSHIP, +] +"""Namespaces every measurement can fill: its score, its score set, and its relation to the request.""" + +MAPPING_DERIVED_NAMESPACES: list[str] = [ + CsvNamespace.REFERENCE_HGVS, + CsvNamespace.VEP, + CsvNamespace.GNOMAD, + CsvNamespace.CLINGEN, +] +"""Namespaces read from a mapped variant, offered whenever the score set has any mapping. + +Scoped to the score set, not the measurement: omitting a column would say "never looked" where the truth +is "looked, found nothing". +""" + +BASE_VARIANT_CSV_NAMESPACES: list[str] = ALWAYS_AVAILABLE_NAMESPACES + MAPPING_DERIVED_NAMESPACES +"""The fixed namespaces a mapped variant's CSV includes. + +``scores_custom`` and ``counts`` are excluded: they vary across score sets, and this export puts one row +per score set, so their columns would be mostly NA. +""" + +EXACT_MATCH_TYPE = "exact" +"""Measurements sharing the requested variant's ClinGen allele ID — currently the only relationship emitted. + +TODO(#784): widen ``_equivalent_measurements`` to nucleotide/amino-acid equivalence once #791 lands +``equivalent_nt``/``equivalent_aa``; ``relationship.match_type`` then takes more than this one value. +See https://github.com/VariantEffect/mavedb-api/issues/784 +""" + + +def _equivalent_measurements( + db: Session, + variant_urn: str, + may_read_score_set: Optional[Callable[[ScoreSet], bool]] = None, +) -> Optional[list[tuple[int, int, int]]]: + """Resolve a variant URN to the measurements the CSV should report. + + Returns ``[(variant_id, mapped_variant_id, score_set_id), ...]``, requested variant first, then every + other current measurement of the same ClinGen allele ordered by score set and variant URN so repeated + downloads are byte-identical. At most one entry per variant, which is what lets the fetch layer + restore this order from variant ids alone. + + Args: + may_read_score_set: when given, drops measurements from score sets the caller may not read. + Applied only to score sets reached by the widening; the caller's own permission check on the + requested variant is not repeated here. + + Returns: + None when the variant has no current mapping, since there is then no allele to expand by. + """ + # Nothing in the schema enforces one current mapping per variant, and every column below follows from + # this pick, so order explicitly: an unordered LIMIT 1 would let one URN download differently twice. + requested = db.scalars( + select(MappedVariant) + .join(MappedVariant.variant) + .where(and_(Variant.urn == variant_urn, MappedVariant.current.is_(True))) + .order_by(MappedVariant.mapped_date.desc(), MappedVariant.id.desc()) + .limit(1) + ).one_or_none() + + if requested is None: + return None + + # TODO(#372): non-null id fields + if not requested.clingen_allele_id: + return [(requested.variant_id, requested.id, requested.variant.score_set_id)] # type: ignore + + # TODO(#784): under the allele-centric (RT) substrate, replace this string match with the + # projection-aware resolver — measurements attach to an Allele and c/g pairs relate through + # `projection_group`. Only this query changes; callers consume (variant_id, mapped_variant_id) pairs. + equivalents = db.execute( + select(MappedVariant.variant_id, MappedVariant.id, ScoreSet.id, ScoreSet.urn, Variant.urn) + .join(MappedVariant.variant) + .join(Variant.score_set) + .where( + and_( + MappedVariant.clingen_allele_id == requested.clingen_allele_id, + MappedVariant.current.is_(True), + # By variant, not by mapping: the anchor already represents this variant, and a second + # current mapping on it is the same measurement again, not an equivalent one. + MappedVariant.variant_id != requested.variant_id, + ) + ) + .order_by(MappedVariant.variant_id, MappedVariant.mapped_date.desc(), MappedVariant.id.desc()) + ).all() + + # One row per variant, picked as the anchor was. A duplicate would defeat the downstream row-order + # restoration, which keys on variant id. TODO(#784): moot once a measurement points at one Allele. + deduplicated: dict[int, Any] = {} + for row in equivalents: + deduplicated.setdefault(row[0], row) + equivalents = list(deduplicated.values()) + + if may_read_score_set is not None and equivalents: + candidate_score_set_ids = {row[2] for row in equivalents} + readable_score_set_ids = { + score_set.id + for score_set in db.scalars(select(ScoreSet).where(ScoreSet.id.in_(candidate_score_set_ids))).all() + if may_read_score_set(score_set) + } + equivalents = [row for row in equivalents if row[2] in readable_score_set_ids] + + equivalents = sorted( + equivalents, + key=lambda row: (score_set_urn_sort_key(row[3]), variant_urn_sort_key(row[4])), + ) + + return [(requested.variant_id, requested.id, requested.variant.score_set_id)] + [ + (row[0], row[1], row[2]) for row in equivalents + ] + + +def _unmapped_variant_namespaces(db: Session, score_set_id: int) -> list[str]: + """The namespaces to offer for a variant that exists but has no current mapping. + + Always score, score set, and relationship; plus the mapping-derived groups when the score set has been + mapped at all, where NA is the honest value for a variant the mapper has not reached. + """ + if score_sets_have_current_mappings(db, [score_set_id]): + return list(BASE_VARIANT_CSV_NAMESPACES) + return list(ALWAYS_AVAILABLE_NAMESPACES) + + +def _latest_clinvar_namespace(db: Session, score_set_ids: list[int]) -> Optional[str]: + """The ClinVar namespace for the most recent release covering these measurements' score sets. + + One release for the whole file rather than each variant's own latest: that keeps the release in the + column header where it is citable, and avoids mixing calls from different releases in one column. + """ + namespaces = clinvar_release_namespaces(db, score_set_ids) + if not namespaces: + return None + + return max(namespaces, key=clinvar_namespace_sort_key) + + +def _annotatable_calibration_namespaces( + db: Session, + score_set_ids: list[int], + viewer: Optional[ScoreCalibrationViewer] = None, +) -> dict[str, ScoreCalibration]: + """Map calibration namespace to calibration, for every calibration eligible to annotate these variants. + + A measurement is only interpretable under its own score set's calibrations, so widening across score + sets widens this set too; a row shows NA under any calibration that does not apply to it. + Research-use-only calibrations are included here but excluded from the default selection. + """ + # Keyed on score sets rather than joined through their variants: `ScoreSet.variants` multiplies the + # join by every variant before DISTINCT collapses it again, for the same result. + calibrations = db.scalars( + select(ScoreCalibration) + .where(ScoreCalibration.score_set_id.in_(score_set_ids)) + .options( + selectinload(ScoreCalibration.functional_classifications).selectinload( + ScoreCalibrationFunctionalClassification.acmg_classification + ), + # Each entry reports the score set it belongs to, so a picker can tell one score set's + # calibrations from another's when the export widens across several. + selectinload(ScoreCalibration.score_set), + ) + ).all() + + namespaces: dict[str, ScoreCalibration] = {} + for calibration in calibration_viewer(viewer).visible(calibrations): + if not calibration.urn: + continue + + # Dropped outright, where the score-set export merely leaves it unchecked: a variant's + # calibrations are scoped to what interprets *this* allele. + if not calibration_can_annotate(calibration): + continue + + namespaces[calibration_namespace_for_urn(str(calibration.urn))] = calibration + + return namespaces + + +def available_variant_csv_namespaces( + db: Session, + variant_urn: str, + may_read_score_set: Optional[Callable[[ScoreSet], bool]] = None, + viewer: Optional[ScoreCalibrationViewer] = None, +) -> list[AvailableCsvNamespaceEntry]: + """Every namespace the variant CSV can serve data for, labeled and grouped for a picker. + + The fixed namespaces, one ``calibration.`` per eligible calibration across the variant's + equivalent measurements, and one ``clinvar.YYYY_MM`` per release covering them. + + Raises: + ValueError: if no variant with *variant_urn* exists. + """ + measurements = _equivalent_measurements(db, variant_urn, may_read_score_set=may_read_score_set) + + if measurements is None: + variant = db.scalars(select(Variant).where(Variant.urn == variant_urn).limit(1)).first() + if variant is None: + raise ValueError(f"variant with URN '{variant_urn}' not found") + + # No mapping on this variant, but its score set may still be mapped, in which case the + # mapping-derived columns are owed with NA rather than omitted. + # TODO(#372): non-null id fields + return [static_namespace_entry(ns) for ns in _unmapped_variant_namespaces(db, int(variant.score_set_id))] # type: ignore + + base_entries = [static_namespace_entry(ns) for ns in BASE_VARIANT_CSV_NAMESPACES] + + score_set_ids = list({score_set_id for _, _, score_set_id in measurements}) + + return ( + base_entries + + calibration_namespace_entries(_annotatable_calibration_namespaces(db, score_set_ids, viewer).values()) + + clinvar_namespace_entries(clinvar_release_namespaces(db, score_set_ids)) + ) + + +def get_variant_csv( + db: Session, + variant_urn: str, + namespaces: Optional[list[str]] = None, + may_read_score_set: Optional[Callable[[ScoreSet], bool]] = None, + viewer: Optional[ScoreCalibrationViewer] = None, + na_rep: str = NA_VALUE, +) -> str: + """Build the clinical CSV for a variant and its equivalent measurements. + + One row per measurement: the requested variant first, then every other current measurement of the + same ClinGen allele across score sets. + + Args: + namespaces: columns to include, in the same vocabulary as the score-set CSV. When omitted, + defaults to the fixed namespaces plus every eligible calibration and the latest ClinVar + release covering these measurements. + + Raises: + ValueError: if no variant with *variant_urn* exists. + """ + measurements = _equivalent_measurements(db, variant_urn, may_read_score_set=may_read_score_set) + + if measurements is None: + return _unmapped_variant_csv(db, variant_urn, namespaces=namespaces, na_rep=na_rep) + + variant_ids = [variant_id for variant_id, _, _ in measurements] + mapped_variant_ids = [mapped_variant_id for _, mapped_variant_id, _ in measurements] + score_set_ids = list({score_set_id for _, _, score_set_id in measurements}) + + calibrations_by_ns = _annotatable_calibration_namespaces(db, score_set_ids, viewer) + + if namespaces is None: + clinvar_namespace = _latest_clinvar_namespace(db, score_set_ids) + # Research-use-only calibrations are offerable but never defaulted to. This export is framed clinically. + default_calibrations = sorted( + namespace for namespace, calibration in calibrations_by_ns.items() if not calibration.research_use_only + ) + resolved_namespaces = ( + BASE_VARIANT_CSV_NAMESPACES + default_calibrations + ([clinvar_namespace] if clinvar_namespace else []) + ) + else: + resolved_namespaces = list(namespaces) + + # Only the required score column is taken, so the score set's own dataset columns are irrelevant. + plan = plan_csv_columns(dataset_columns={}, namespaces=resolved_namespaces) + columns = plan.namespaced_columns + + fetched = fetch_variant_csv_data( + db, + columns, + plan.clinvar_namespaces, + variant_ids=variant_ids, + mapped_variant_ids=mapped_variant_ids, + ) + + mappings = fetched.mappings or [None] * len(fetched.variants) + + rows = variants_to_csv_rows( + fetched.variants, + columns, + mappings=fetched.mappings, + gnomad_data=fetched.gnomad_data, + clinvar_data_by_ns=fetched.clinvar_per_variant, + annotations_by_ns=annotations_for_rows(db, fetched.variants, mappings, plan.calibration_namespaces, viewer), + match_types=[EXACT_MATCH_TYPE] * len(fetched.variants), + na_rep=na_rep, + namespaced=True, + ) + + return rows_to_csv(rows, assemble_csv_headers(columns, namespaced=True)) + + +def _unmapped_variant_csv( + db: Session, + variant_urn: str, + namespaces: Optional[list[str]] = None, + na_rep: str = NA_VALUE, +) -> str: + """Build the single-row CSV for a variant that exists but has no current mapping. + + Without a mapping there are no coordinates, no external annotations, no allele to find equivalents by, + and nothing for the annotation layer to flatten. Identity, score, and provenance still resolve, so the + download succeeds rather than 404ing on a variant that genuinely exists. + """ + variant = db.scalars( + select(Variant) + .where(Variant.urn == variant_urn) + .options( + selectinload(Variant.score_set).selectinload(ScoreSet.target_genes), + ) + ).one_or_none() + + if variant is None: + raise ValueError(f"variant with URN '{variant_urn}' not found") + + plan = plan_csv_columns( + dataset_columns={}, + namespaces=( + # TODO(#372): non-null id fields + list(namespaces) if namespaces is not None else _unmapped_variant_namespaces(db, int(variant.score_set_id)) # type: ignore + ), + ) + columns = plan.namespaced_columns + + rows: list[dict[str, Any]] = list( + variants_to_csv_rows( + [variant], + columns, + match_types=[EXACT_MATCH_TYPE], + na_rep=na_rep, + namespaced=True, + ) + ) + return rows_to_csv(rows, assemble_csv_headers(columns, namespaced=True)) diff --git a/src/mavedb/lib/mave/utils.py b/src/mavedb/lib/mave/utils.py index dd6b75916..f446150dc 100644 --- a/src/mavedb/lib/mave/utils.py +++ b/src/mavedb/lib/mave/utils.py @@ -3,6 +3,11 @@ import pandas as pd NA_VALUE = "NA" +"""The MAVE convention for a missing value: written by the CSV exports, recognised by the CSV ingest. + +Shared vocabulary rather than an export detail, which is why it stays here while the export-side null +predicate lives with the exporter in ``lib/csv/columns.py``. +""" NULL_VALUES = ("", "na", "nan", "nil", "none", "null", "n/a", "undefined", NA_VALUE) diff --git a/src/mavedb/lib/score_sets.py b/src/mavedb/lib/score_sets.py index 8e3c8debb..698bc515a 100644 --- a/src/mavedb/lib/score_sets.py +++ b/src/mavedb/lib/score_sets.py @@ -1,17 +1,14 @@ -import csv -import io import logging -import re from collections import Counter, defaultdict from operator import attrgetter -from typing import TYPE_CHECKING, Any, BinaryIO, Iterable, List, Optional, Sequence +from typing import TYPE_CHECKING, BinaryIO, Optional, Sequence import numpy as np import pandas as pd from pandas.testing import assert_index_equal -from sqlalchemy import Integer, and_, cast, func, or_, select -from sqlalchemy.orm import Query, Session, aliased, contains_eager, joinedload, selectinload +from sqlalchemy import and_, func, or_, select from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Query, Session, aliased, contains_eager, joinedload, selectinload from mavedb.lib.exceptions import ValidationError from mavedb.lib.logging.context import logging_context, save_to_logging_context @@ -19,7 +16,6 @@ HGVS_NT_COLUMN, HGVS_PRO_COLUMN, HGVS_SPLICE_COLUMN, - REQUIRED_SCORE_COLUMN, VARIANT_COUNT_DATA, VARIANT_SCORE_DATA, ) @@ -27,8 +23,6 @@ from mavedb.lib.permissions import Action, has_permission from mavedb.lib.types.authentication import UserData from mavedb.lib.validation.constants.general import null_values_list -from mavedb.lib.validation.utilities import is_null as validate_is_null -from mavedb.lib.variants import get_digest_from_post_mapped, get_hgvs_from_post_mapped, is_hgvs_g, is_hgvs_p from mavedb.models.contributor import Contributor from mavedb.models.controlled_keyword import ControlledKeyword from mavedb.models.doi_identifier import DoiIdentifier @@ -38,9 +32,6 @@ from mavedb.models.experiment_controlled_keyword import ExperimentControlledKeywordAssociation from mavedb.models.experiment_publication_identifier import ExperimentPublicationIdentifierAssociation from mavedb.models.experiment_set import ExperimentSet -from mavedb.models.clinical_control import ClinicalControl -from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table -from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.mapped_variant import MappedVariant from mavedb.models.publication_identifier import PublicationIdentifier from mavedb.models.refseq_identifier import RefseqIdentifier @@ -57,7 +48,7 @@ from mavedb.models.uniprot_offset import UniprotOffset from mavedb.models.user import User from mavedb.models.variant import Variant -from mavedb.view_models.search import ScoreSetsSearch, ControlledKeywordFilterOption +from mavedb.view_models.search import ControlledKeywordFilterOption, ScoreSetsSearch if TYPE_CHECKING: from mavedb.lib.permissions import Action @@ -66,10 +57,6 @@ logger = logging.getLogger(__name__) -# Pattern for ClinVar-versioned namespaces of the form "clinvar.YEAR_MONTH", -# e.g. "clinvar.2024_01" for January 2024. -CLINVAR_NS_PATTERN = re.compile(r"^clinvar\.(\d+)_(0[1-9]|1[0-2])$") - class HGVSColumns: NUCLEOTIDE: str = "hgvs_nt" # dataset.constants.hgvs_nt_column @@ -587,258 +574,6 @@ def get_current_mapped_variants_for_annotation(db: Session, score_set: ScoreSet) ) -def get_score_set_variants_as_csv( - db: Session, - score_set: ScoreSet, - namespaces: List[str], - namespaced: Optional[bool] = None, - start: Optional[int] = None, - limit: Optional[int] = None, - drop_na_columns: Optional[bool] = None, - include_custom_columns: Optional[bool] = True, - include_post_mapped_hgvs: Optional[bool] = False, -) -> str: - """ - Get the variant data from a score set as a CSV string. - - Parameters - __________ - db : Session - The database session to use. - score_set : ScoreSet - The score set to get the variants from. - namespaces : List[str] - The namespaces for data: "scores", "counts", "vep", "gnomad", "clingen", and/or - ClinVar-versioned namespaces of the form "clinvar.YEAR_MONTH" (e.g. "clinvar.2024_01" - for January 2024, which joins on db_name="ClinVar" and db_version="01_2024"). - namespaced: Optional[bool] = None - Whether namespace the columns or not. - start : int, optional - The index to start from. If None, starts from the beginning. - limit : int, optional - The maximum number of variants to return. If None, returns all variants. - drop_na_columns : bool, optional - Whether to drop columns that contain only NA values. Defaults to False. - include_custom_columns : bool, optional - Whether to include custom columns defined in the score set. Defaults to True. - include_post_mapped_hgvs : bool, optional - Whether to include post-mapped HGVS notations and VEP functional consequence in the output. Defaults to False. If True, the output will include - columns for post-mapped HGVS genomic (g.) and protein (p.) notations, and VEP functional consequence. - - Returns - _______ - str - The CSV string containing the variant data. - """ - assert type(score_set.dataset_columns) is dict - namespaced_score_set_columns: dict[str, list[str]] = { - "core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"], - "mavedb": [], - } - if include_post_mapped_hgvs: - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_g") - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_p") - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_c") - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_at_assay_level") - namespaced_score_set_columns["mavedb"].append("post_mapped_vrs_digest") - for namespace in namespaces: - namespaced_score_set_columns[namespace] = [] - - if include_custom_columns: - if "scores" in namespaced_score_set_columns: - namespaced_score_set_columns["scores"] = [ - col for col in [str(x) for x in list(score_set.dataset_columns.get("score_columns", []))] - ] - if "counts" in namespaced_score_set_columns: - namespaced_score_set_columns["counts"] = [ - col for col in [str(x) for x in list(score_set.dataset_columns.get("count_columns", []))] - ] - elif "scores" in namespaced_score_set_columns: - namespaced_score_set_columns["scores"].append(REQUIRED_SCORE_COLUMN) - if "vep" in namespaced_score_set_columns: - namespaced_score_set_columns["vep"].append("vep_functional_consequence") - if "gnomad" in namespaced_score_set_columns: - namespaced_score_set_columns["gnomad"].append("gnomad_af") - if "clingen" in namespaced_score_set_columns: - namespaced_score_set_columns["clingen"].append("clingen_allele_id") - - # Parse ClinVar-versioned namespaces of the form "clinvar.YEAR_MONTH". - # The corresponding db_version stored in clinical_controls is "MONTH_YEAR". - clinvar_namespaces: dict[str, str] = {} # namespace -> db_version (MONTH_YEAR) - for ns in namespaces: - m = CLINVAR_NS_PATTERN.match(ns) - if m: - year, month = m.group(1), m.group(2) - db_version = f"{month}_{year}" - clinvar_namespaces[ns] = db_version - namespaced_score_set_columns[ns] = ["clinical_significance", "clinical_review_status"] - - need_mappings = ( - include_post_mapped_hgvs - or "clingen" in namespaces - or "vep" in namespaces - or "gnomad" in namespaces - or bool(clinvar_namespaces) - ) - need_gnomad = "gnomad" in namespaces - - variants: list[Variant] = [] - mappings: Optional[list[Optional[MappedVariant]]] = [] if need_mappings else None - gnomad_data: Optional[list[Optional[GnomADVariant]]] = [] if need_gnomad else None - - select_columns: list[Any] = [Variant] - if need_mappings: - select_columns.append(MappedVariant) - if need_gnomad: - select_columns.append(GnomADVariant) - - query = ( - select(*select_columns) - .where(Variant.score_set_id == score_set.id) - .order_by(cast(func.split_part(Variant.urn, "#", 2), Integer)) - ) - - if need_mappings: - query = query.join( - MappedVariant, - and_(Variant.id == MappedVariant.variant_id, MappedVariant.current.is_(True)), - isouter=True, - ) - - if need_gnomad: - query = query.join( - MappedVariant.gnomad_variants.of_type(GnomADVariant), - isouter=True, - ).where( - or_( - and_(GnomADVariant.db_name == "gnomAD", GnomADVariant.db_version == "v4.1"), - GnomADVariant.id.is_(None), - ) - ) - - if start: - query = query.offset(start) - if limit: - query = query.limit(limit) - - result = db.execute(query).all() - - for row in result: - variant = row[0] - variants.append(variant) - - if need_mappings and mappings is not None: - mappings.append(row[1]) - - if need_gnomad and gnomad_data is not None: - idx = 2 if need_mappings else 1 - gnomad_data.append(row[idx]) - - # For each ClinVar namespace, fetch a mapping from mapped_variant_id to ClinicalControl. - clinvar_data_map: dict[str, dict[int, Optional[ClinicalControl]]] = {} - if clinvar_namespaces and mappings is not None: - mv_ids = [m.id for m in mappings if m is not None] - for ns, db_version in clinvar_namespaces.items(): - mv_to_cc: dict[int, Optional[ClinicalControl]] = {} - if mv_ids: - aliased_cc = aliased(ClinicalControl) - cc_query = ( - select( - mapped_variants_clinical_controls_association_table.c.mapped_variant_id, - aliased_cc, - ) - .join( - aliased_cc, - mapped_variants_clinical_controls_association_table.c.clinical_control_id == aliased_cc.id, - ) - .where( - and_( - mapped_variants_clinical_controls_association_table.c.mapped_variant_id.in_(mv_ids), - aliased_cc.db_name == "ClinVar", - aliased_cc.db_version == db_version, - ) - ) - ) - for mv_id, cc in db.execute(cc_query).all(): - mv_to_cc[mv_id] = cc - clinvar_data_map[ns] = mv_to_cc - - # Build per-variant ClinVar lookup (list indexed in parallel with variants). - clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] = None - if clinvar_namespaces and mappings is not None: - clinvar_per_variant = [] - for mapping in mappings: - row_clinvar: dict[str, Optional[ClinicalControl]] = {} - for ns, mv_to_cc in clinvar_data_map.items(): - if mapping is not None and mapping.id is not None: - row_clinvar[ns] = mv_to_cc.get(mapping.id) - else: - row_clinvar[ns] = None - clinvar_per_variant.append(row_clinvar) - - rows_data = variants_to_csv_rows( - variants, - columns=namespaced_score_set_columns, - namespaced=namespaced, - mappings=mappings, - gnomad_data=gnomad_data, - clinvar_data_by_ns=clinvar_per_variant, - ) # type: ignore - - rows_columns = [] - for namespace, cols in namespaced_score_set_columns.items(): - for col in cols: - if CLINVAR_NS_PATTERN.match(namespace): - # ClinVar versioned namespaces always include the full namespace prefix - # to avoid column-name collisions when multiple versions are requested. - rows_columns.append(f"{namespace}.{col}") - elif namespaced and namespace not in ["core", "mavedb"]: - rows_columns.append(f"{namespace}.{col}") - elif namespaced and namespace == "mavedb": - rows_columns.append(f"mavedb.{col}") - else: - rows_columns.append(col) - - if drop_na_columns: - rows_data, rows_columns = drop_na_columns_from_csv_file_rows(rows_data, rows_columns) - - stream = io.StringIO() - writer = csv.DictWriter(stream, fieldnames=rows_columns, quoting=csv.QUOTE_MINIMAL) - writer.writeheader() - writer.writerows(rows_data) - return stream.getvalue() - - -def drop_na_columns_from_csv_file_rows( - rows_data: Iterable[dict[str, Any]], columns: list[str] -) -> tuple[list[dict[str, Any]], list[str]]: - """Process rows_data for downloadable CSV by removing empty columns.""" - # Convert map to list. - rows_data = list(rows_data) - columns_to_check = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] - columns_to_remove = [] - - # Check if all values in a column are None or "NA" - for col in columns_to_check: - if all(validate_is_null(row[col]) for row in rows_data): - columns_to_remove.append(col) - for row in rows_data: - row.pop(col, None) # Remove column from each row - - # Remove these columns from the header list - columns = [col for col in columns if col not in columns_to_remove] - return rows_data, columns - - -null_values_re = re.compile(r"\s+|none|nan|na|undefined|n/a|null|nil", flags=re.IGNORECASE) - - -def is_null(value): - """Return True if a string represents a null value.""" - value = str(value).strip().lower() - return null_values_re.fullmatch(value) or not value - - def is_replaces_id_unique_violation(exc: IntegrityError) -> bool: """ Return True if the IntegrityError was caused by the unique constraint on score_set.replaces_id. @@ -852,203 +587,6 @@ def is_replaces_id_unique_violation(exc: IntegrityError) -> bool: return "replaces_id" in detail -def variant_to_csv_row( - variant: Variant, - columns: dict[str, list[str]], - mapping: Optional[MappedVariant] = None, - gnomad_data: Optional[GnomADVariant] = None, - clinvar_data_by_ns: Optional[dict[str, Optional[ClinicalControl]]] = None, - namespaced: Optional[bool] = None, - na_rep="NA", -) -> dict[str, Any]: - """ - Format a variant into a containing the keys specified in `columns`. - - Parameters - ---------- - variant : variant.models.Variant - List of variants. - columns : list[str] - Columns to serialize. - namespaced: Optional[bool] = None - Namespace the columns or not. - mapping : variant.models.MappedVariant, optional - Mapped variant corresponding to the variant. - gnomad_data : variant.models.GnomADVariant, optional - gnomAD variant data corresponding to the variant. - clinvar_data_by_ns : dict[str, Optional[ClinicalControl]], optional - Per-variant ClinVar data keyed by namespace (e.g. "clinvar.2024_01"). - na_rep : str - String to represent null values. - - Returns - ------- - dict[str, Any] - """ - row: dict[str, Any] = {} - # Handle each column key explicitly as part of its namespace. - for column_key in columns.get("core", []): - if column_key == "hgvs_nt": - value = str(variant.hgvs_nt) - elif column_key == "hgvs_pro": - value = str(variant.hgvs_pro) - elif column_key == "hgvs_splice": - value = str(variant.hgvs_splice) - elif column_key == "accession": - value = str(variant.urn) - if is_null(value): - value = na_rep - - # export columns in the `core` namespace without a namespace - row[column_key] = value - for column_key in columns.get("mavedb", []): - if column_key == "post_mapped_hgvs_g": - value = str(mapping.hgvs_g) if mapping and mapping.hgvs_g else na_rep - if value == na_rep: - fallback_hgvs = ( - get_hgvs_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None - ) - if fallback_hgvs is not None and is_hgvs_g(fallback_hgvs): - value = fallback_hgvs - else: - value = na_rep - - elif column_key == "post_mapped_hgvs_p": - value = str(mapping.hgvs_p) if mapping and mapping.hgvs_p else na_rep - if value == na_rep: - fallback_hgvs = ( - get_hgvs_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None - ) - if fallback_hgvs is not None and is_hgvs_p(fallback_hgvs): - value = fallback_hgvs - else: - value = na_rep - - elif column_key == "post_mapped_hgvs_c": - value = str(mapping.hgvs_c) if mapping and mapping.hgvs_c else na_rep - elif column_key == "post_mapped_hgvs_at_assay_level": - value = str(mapping.hgvs_assay_level) if mapping and mapping.hgvs_assay_level else na_rep - elif column_key == "post_mapped_vrs_digest": - digest = get_digest_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None - value = digest if digest is not None else na_rep - if is_null(value): - value = na_rep - key = f"mavedb.{column_key}" if namespaced else column_key - row[key] = value - for column_key in columns.get("vep", []): - if column_key == "vep_functional_consequence": - vep_functional_consequence = mapping.vep_functional_consequence if mapping else None - if vep_functional_consequence is not None: - value = vep_functional_consequence - else: - value = na_rep - key = f"vep.{column_key}" if namespaced else column_key - row[key] = value - for column_key in columns.get("scores", []): - parent = variant.data.get("score_data") if variant.data else None - value = str(parent.get(column_key)) if parent else na_rep - if is_null(value): - value = na_rep - key = f"scores.{column_key}" if namespaced else column_key - row[key] = value - for column_key in columns.get("counts", []): - parent = variant.data.get("count_data") if variant.data else None - value = str(parent.get(column_key)) if parent else na_rep - if is_null(value): - value = na_rep - key = f"counts.{column_key}" if namespaced else column_key - row[key] = value - for column_key in columns.get("gnomad", []): - if column_key == "gnomad_af": - gnomad_af = gnomad_data.allele_frequency if gnomad_data else None - if gnomad_af is not None: - value = str(gnomad_af) - else: - value = na_rep - key = f"gnomad.{column_key}" if namespaced else column_key - row[key] = value - for column_key in columns.get("clingen", []): - if column_key == "clingen_allele_id": - clingen_allele_id = mapping.clingen_allele_id if mapping else None - if clingen_allele_id is not None: - value = str(clingen_allele_id) - else: - value = na_rep - key = f"clingen.{column_key}" if namespaced else column_key - row[key] = value - # Handle ClinVar-versioned namespaces (e.g. "clinvar.2024_01"). - # These always use the full "namespace.column" key regardless of the namespaced flag - # to avoid collisions when multiple versions are requested. - for namespace_key, namespace_cols in columns.items(): - if not CLINVAR_NS_PATTERN.match(namespace_key): - continue - clinvar_entry = (clinvar_data_by_ns or {}).get(namespace_key) - for column_key in namespace_cols: - if column_key == "clinical_significance": - value = str(clinvar_entry.clinical_significance) if clinvar_entry else na_rep - elif column_key == "clinical_review_status": - value = str(clinvar_entry.clinical_review_status) if clinvar_entry else na_rep - else: - value = na_rep - if is_null(value): - value = na_rep - row[f"{namespace_key}.{column_key}"] = value - return row - - -def variants_to_csv_rows( - variants: Sequence[Variant], - columns: dict[str, list[str]], - mappings: Optional[Sequence[Optional[MappedVariant]]] = None, - gnomad_data: Optional[Sequence[Optional[GnomADVariant]]] = None, - clinvar_data_by_ns: Optional[Sequence[Optional[dict[str, Optional[ClinicalControl]]]]] = None, - namespaced: Optional[bool] = None, - na_rep="NA", -) -> Iterable[dict[str, Any]]: - """ - Format each variant into a dictionary row containing the keys specified in `columns`. - - Parameters - ---------- - variants : list[variant.models.Variant] - List of variants. - columns : list[str] - Columns to serialize. - namespaced: Optional[bool] = None - Namespace the columns or not. - mappings : list[Optional[variant.models.MappedVariant]], optional - List of mapped variants corresponding to the variants. - gnomad_data : list[Optional[variant.models.GnomADVariant]], optional - List of gnomAD variant data corresponding to the variants. - clinvar_data_by_ns : list[Optional[dict[str, Optional[ClinicalControl]]]], optional - Per-variant ClinVar data keyed by namespace (e.g. "clinvar.2024_01"). - na_rep : str - String to represent null values. - - Returns - ------- - list[dict[str, Any]] - """ - n = len(variants) - _mappings: Sequence[Optional[MappedVariant]] = mappings if mappings is not None else [None] * n - _gnomad: Sequence[Optional[GnomADVariant]] = gnomad_data if gnomad_data is not None else [None] * n - _clinvar: Sequence[Optional[dict[str, Optional[ClinicalControl]]]] = ( - clinvar_data_by_ns if clinvar_data_by_ns is not None else [None] * n - ) - return map( - lambda t: variant_to_csv_row( - t[0], - columns, - mapping=t[1], - gnomad_data=t[2], - clinvar_data_by_ns=t[3], - namespaced=namespaced, - na_rep=na_rep, - ), - zip(variants, _mappings, _gnomad, _clinvar), - ) - - def find_meta_analyses_for_score_sets(db: Session, urns: list[str]) -> list[ScoreSet]: """ Find all score sets that are meta-analyses for a specified collection of other score sets. diff --git a/src/mavedb/lib/urns.py b/src/mavedb/lib/urns.py index 55a59e707..46be37a10 100644 --- a/src/mavedb/lib/urns.py +++ b/src/mavedb/lib/urns.py @@ -1,11 +1,16 @@ import logging import re import string +from typing import Optional from uuid import uuid4 from sqlalchemy import func from sqlalchemy.orm import Session +from mavedb.lib.validation.urn_re import ( + MAVEDB_EXPERIMENT_SET_URN_DIGITS, + MAVEDB_URN_NAMESPACE, +) from mavedb.models.experiment import Experiment from mavedb.models.experiment_set import ExperimentSet from mavedb.models.score_set import ScoreSet @@ -175,3 +180,55 @@ def generate_job_run_urn(): :return: A new job run URN """ return f"urn:mavedb:job-{uuid4()}" + + +# MaveDB URNs do not sort into assignment order as strings: score-set and variant suffixes are unpadded +# (`-a-10` < `-a-2`, `#10` < `#2`) and experiment suffixes run a..z then aa..az (`aa` < `b`). Only the +# experiment-set digits are padded, which is why a lexical sort looks right until double digits. The keys +# below are the read side of the rule `generate_experiment_urn` already applies when assigning. + + +_SCORE_SET_URN_PARTS_RE = re.compile( + rf"^(?Purn:{MAVEDB_URN_NAMESPACE}:\d{{{MAVEDB_EXPERIMENT_SET_URN_DIGITS}}})" + r"-(?P[a-z]+|0)" + r"-(?P[1-9]\d*)$" +) + +_VARIANT_URN_PARTS_RE = re.compile(r"^(?P.+)#(?P[1-9]\d*)$") + +_UNPARSED = 1 +"""Leading element for an undecomposable URN, so it sorts after every well-formed one. + +Unpublished records carry ``tmp:`` URNs; they still order stably, by the URN itself. Returning a key +rather than raising keeps a temporary URN from turning into a query error. +""" + +_PARSED = 0 + + +def score_set_urn_sort_key(urn: Optional[str]) -> tuple[int, str, int, str, int]: + """Sort key ordering score set URNs the way their parts were assigned.""" + if not urn: + return (_UNPARSED, "", 0, "", 0) + + match = _SCORE_SET_URN_PARTS_RE.match(urn) + if match is None: + return (_UNPARSED, urn, 0, "", 0) + + experiment = match["experiment"] + return (_PARSED, match["experiment_set"], len(experiment), experiment, int(match["score_set"])) + + +def variant_urn_sort_key(urn: Optional[str]) -> tuple[int, str, int]: + """Sort key ordering variant URNs by score set, then by numeric suffix. + + ``...#10`` is the tenth variant of a score set, not something between ``#1`` and ``#2``. + """ + if not urn: + return (_UNPARSED, "", 0) + + match = _VARIANT_URN_PARTS_RE.match(urn) + if match is None: + return (_UNPARSED, urn, 0) + + return (_PARSED, match["score_set"], int(match["number"])) diff --git a/src/mavedb/lib/validation/urn_re.py b/src/mavedb/lib/validation/urn_re.py index 82feb19a2..dddc9d142 100644 --- a/src/mavedb/lib/validation/urn_re.py +++ b/src/mavedb/lib/validation/urn_re.py @@ -32,6 +32,10 @@ MAVEDB_COLLECTION_URN_PATTERN = r"urn:mavedb:collection-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" MAVEDB_COLLECTION_URN_RE = re.compile(MAVEDB_COLLECTION_URN_PATTERN) +# Score calibration URN +MAVEDB_CALIBRATION_URN_PATTERN = r"urn:mavedb:calibration-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" +MAVEDB_CALIBRATION_URN_RE = re.compile(MAVEDB_CALIBRATION_URN_PATTERN) + # Any URN MAVEDB_ANY_URN_PATTERN = "|".join( [ diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 4d1a30ad5..de0cda700 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -53,16 +53,23 @@ from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.lib.score_calibrations import create_score_calibration +from mavedb.lib.csv.deprecated_params import ( + DROP_NA_COLUMNS_DESCRIPTION, + INCLUDE_CUSTOM_COLUMNS_DESCRIPTION, + INCLUDE_POST_MAPPED_HGVS_DESCRIPTION, + resolve_deprecated_csv_params, +) +from mavedb.lib.csv.namespaces import CSV_NAMESPACES_PARAM_DESCRIPTION, CsvNamespaceStr +from mavedb.view_models.csv_namespace import AvailableCsvNamespace +from mavedb.lib.csv.columns import variants_to_csv_rows +from mavedb.lib.csv.score_set import available_score_set_csv_namespaces, get_score_set_variants_as_csv from mavedb.lib.score_sets import ( - CLINVAR_NS_PATTERN, csv_data_to_df, fetch_score_set_search_filter_options, find_meta_analyses_for_experiment_sets, get_current_mapped_variants_for_annotation, - get_score_set_variants_as_csv, is_replaces_id_unique_violation, refresh_variant_urns, - variants_to_csv_rows, ) from mavedb.lib.score_sets import ( search_score_sets as _search_score_sets, @@ -909,6 +916,58 @@ async def show_score_set( return _score_set_response(item, principal) +@router.get( + "/score-sets/{urn}/csv-namespaces", + status_code=200, + response_model=list[AvailableCsvNamespace], + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="List the CSV column namespaces this score set has data for", +) +def get_score_set_csv_namespaces( + *, + urn: str, + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), +) -> Any: + """ + List the CSV column namespaces this score set has data for, labeled and grouped for a picker. + + Each entry's `namespace` is a value accepted by the `namespaces` parameter of the CSV endpoints. + Deliberately a separate request rather than a field on the score set: it costs several queries and is + only needed when a user opens a download dialog, so it should not sit on the score-set page's + critical path. + + Parameters + __________ + urn : str + The URN of the score set to inspect. + db : Session + The database session to use. + user_data : Optional[UserData] + The user data of the current user. If None, no user-specific permissions are checked. + + Returns + _______ + list[AvailableCsvNamespace] + The namespaces with data, each with a human-readable label and group. + """ + save_to_logging_context({"requested_resource": urn, "resource_property": "csv-namespaces"}) + + score_set = db.query(ScoreSet).filter(ScoreSet.urn == urn).first() + if not score_set: + logger.info(msg="Could not fetch CSV namespaces; No such score set exists.", extra=logging_context()) + raise HTTPException(status_code=404, detail=f"score set with URN '{urn}' not found") + + assert_permission(user_data, score_set, Action.READ) + + return available_score_set_csv_namespaces( + db, + score_set, + viewer=principal.viewer_for(ScoreCalibrationViewer), + ) + + @router.get( "/score-sets/{urn}/variants/data", status_code=200, @@ -928,19 +987,21 @@ def get_score_set_variants_csv( urn: str, start: int = Query(default=None, description="Start index for pagination"), limit: int = Query(default=None, description="Maximum number of variants to return"), - namespaces: List[str] = Query( + namespaces: List[CsvNamespaceStr] = Query( default=["scores"], - description=( - 'One or more data types to include: "scores", "counts", "vep", "gnomad", "clingen", ' - 'and/or ClinVar-versioned namespaces of the form "clinvar.YEAR_MONTH" ' - '(e.g. "clinvar.2024_01" for January 2024).' - ), + description=CSV_NAMESPACES_PARAM_DESCRIPTION, + ), + drop_unused_hgvs_columns: Optional[bool] = None, + drop_na_columns: Optional[bool] = Query(default=None, deprecated=True, description=DROP_NA_COLUMNS_DESCRIPTION), + include_post_mapped_hgvs: Optional[bool] = Query( + default=None, deprecated=True, description=INCLUDE_POST_MAPPED_HGVS_DESCRIPTION + ), + include_custom_columns: Optional[bool] = Query( + default=None, deprecated=True, description=INCLUDE_CUSTOM_COLUMNS_DESCRIPTION ), - drop_na_columns: Optional[bool] = None, - include_custom_columns: Optional[bool] = None, - include_post_mapped_hgvs: Optional[bool] = None, db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), ) -> Any: """ Return tabular variant data from a score set, identified by URN, in CSV format. @@ -958,11 +1019,19 @@ def get_score_set_variants_csv( The maximum number of variants to return. If None, returns all variants. namespaces: List[str] The namespaces of all columns except for accession, hgvs_nt, hgvs_pro, and hgvs_splice. - Supported values: "scores", "counts", "vep", "gnomad", "clingen", and ClinVar-versioned - namespaces of the form "clinvar.YEAR_MONTH" (e.g. "clinvar.2024_01" for January 2024). - Multiple ClinVar namespaces with different YEAR_MONTH values may be requested simultaneously. + Supported values: "scores" (the required score column), "scores_custom" (the investigator's + remaining score columns, emitted under the "scores" prefix), "counts", "mavedb", "vep", "gnomad", + "clingen", "score_set", and ClinVar- and calibration-parameterized namespaces. Multiple ClinVar + and calibration namespaces may be requested simultaneously. + drop_unused_hgvs_columns : bool, optional + Whether to omit the HGVS coordinate columns this score set does not use, e.g. hgvs_nt for a + protein-only score set. Defaults to False. drop_na_columns : bool, optional - Whether to drop columns that contain only NA values. Defaults to False. + Deprecated spelling of drop_unused_hgvs_columns, accepted for one release. + include_post_mapped_hgvs : bool, optional + Deprecated: equivalent to requesting the "mavedb" namespace. Accepted for one release. + include_custom_columns : bool, optional + Deprecated: equivalent to requesting the "scores_custom" namespace. Accepted for one release. db : Session The database session to use. user_data : Optional[UserData] @@ -973,13 +1042,23 @@ def get_score_set_variants_csv( str The CSV string containing the variant data. """ + deprecated = resolve_deprecated_csv_params( + namespaces=namespaces, + drop_unused_hgvs_columns=drop_unused_hgvs_columns, + drop_na_columns=drop_na_columns, + include_post_mapped_hgvs=include_post_mapped_hgvs, + include_custom_columns=include_custom_columns, + ) + namespaces = deprecated.namespaces + drop_unused_hgvs_columns = deprecated.drop_unused_hgvs_columns + save_to_logging_context( { "requested_resource": urn, "resource_property": "scores", "start": start, "limit": limit, - "drop_na_columns": drop_na_columns, + "drop_unused_hgvs_columns": drop_unused_hgvs_columns, } ) @@ -990,21 +1069,6 @@ def get_score_set_variants_csv( logger.info(msg="Could not fetch scores with non-positive limit.", extra=logging_context()) raise HTTPException(status_code=422, detail="Limit must be positive") - _VALID_STATIC_NAMESPACES = {"scores", "counts", "vep", "gnomad", "clingen"} - invalid_namespaces = [ - ns for ns in namespaces if ns not in _VALID_STATIC_NAMESPACES and not CLINVAR_NS_PATTERN.match(ns) - ] - if invalid_namespaces: - raise HTTPException( - status_code=422, - detail=( - f"Invalid namespace(s): {invalid_namespaces}. " - 'Each namespace must be one of "scores", "counts", "vep", "gnomad", "clingen", ' - 'or a ClinVar-versioned namespace of the form "clinvar.YEAR_MM" ' - '(e.g. "clinvar.2024_01" for January 2024).' - ), - ) - score_set = db.query(ScoreSet).filter(ScoreSet.urn == urn).first() if not score_set: logger.info(msg="Could not fetch the requested scores; No such score set exists.", extra=logging_context()) @@ -1019,11 +1083,12 @@ def get_score_set_variants_csv( True, start, limit, - drop_na_columns, - include_custom_columns, - include_post_mapped_hgvs, + drop_unused_hgvs_columns, + # Asked separately from the score set: a private calibration is readable only by its owner, + # investigator contributors, or an admin, whoever can read the score set. + viewer=principal.viewer_for(ScoreCalibrationViewer), ) - return StreamingResponse(iter([csv_str]), media_type="text/csv") + return StreamingResponse(iter([csv_str]), media_type="text/csv", headers=deprecated.response_headers) @router.get( @@ -1045,7 +1110,8 @@ def get_score_set_scores_csv( urn: str, start: int = Query(default=None, description="Start index for pagination"), limit: int = Query(default=None, description="Number of variants to return"), - drop_na_columns: Optional[bool] = None, + drop_unused_hgvs_columns: Optional[bool] = None, + drop_na_columns: Optional[bool] = Query(default=None, deprecated=True, description=DROP_NA_COLUMNS_DESCRIPTION), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), ) -> Any: @@ -1057,6 +1123,11 @@ def get_score_set_scores_csv( /score-sets/{urn}/scores?start=0&limit=100 /score-sets/{urn}/scores?start=100 """ + deprecated = resolve_deprecated_csv_params( + drop_unused_hgvs_columns=drop_unused_hgvs_columns, drop_na_columns=drop_na_columns + ) + drop_unused_hgvs_columns = deprecated.drop_unused_hgvs_columns + save_to_logging_context( { "requested_resource": urn, @@ -1080,8 +1151,12 @@ def get_score_set_scores_csv( assert_permission(user_data, score_set, Action.READ) - csv_str = get_score_set_variants_as_csv(db, score_set, ["scores"], False, start, limit, drop_na_columns) - return StreamingResponse(iter([csv_str]), media_type="text/csv") + # Both score namespaces: this endpoint has always returned every score column the investigator + # uploaded, and `scores` alone is now just the required one. + csv_str = get_score_set_variants_as_csv( + db, score_set, ["scores", "scores_custom"], False, start, limit, drop_unused_hgvs_columns + ) + return StreamingResponse(iter([csv_str]), media_type="text/csv", headers=deprecated.response_headers) @router.get( @@ -1103,7 +1178,8 @@ async def get_score_set_counts_csv( urn: str, start: int = Query(default=None, description="Start index for pagination"), limit: int = Query(default=None, description="Number of variants to return"), - drop_na_columns: Optional[bool] = None, + drop_unused_hgvs_columns: Optional[bool] = None, + drop_na_columns: Optional[bool] = Query(default=None, deprecated=True, description=DROP_NA_COLUMNS_DESCRIPTION), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), ) -> Any: @@ -1115,6 +1191,11 @@ async def get_score_set_counts_csv( /score-sets/{urn}/counts?start=0&limit=100 /score-sets/{urn}/counts?start=100 """ + deprecated = resolve_deprecated_csv_params( + drop_unused_hgvs_columns=drop_unused_hgvs_columns, drop_na_columns=drop_na_columns + ) + drop_unused_hgvs_columns = deprecated.drop_unused_hgvs_columns + save_to_logging_context( { "requested_resource": urn, @@ -1138,8 +1219,8 @@ async def get_score_set_counts_csv( assert_permission(user_data, score_set, Action.READ) - csv_str = get_score_set_variants_as_csv(db, score_set, ["counts"], False, start, limit, drop_na_columns) - return StreamingResponse(iter([csv_str]), media_type="text/csv") + csv_str = get_score_set_variants_as_csv(db, score_set, ["counts"], False, start, limit, drop_unused_hgvs_columns) + return StreamingResponse(iter([csv_str]), media_type="text/csv", headers=deprecated.response_headers) @router.get( @@ -1210,6 +1291,15 @@ def _stream_generated_annotations(mapped_variants, annotation_function): except MappingDataDoesntExistException: logger.debug(f"Mapping data does not exist for variant {mv.variant.urn}.") annotation = None + except Exception: + # Raising here would end the body mid-stream. The 200 and its headers went out with the first + # chunk, so the client has no way to be told and simply receives a short file. Report the + # variant as unannotated and keep going, so one bad variant cannot truncate a whole download. + logger.exception( + f"Failed to annotate variant {mv.variant.urn}; streaming it as unannotated.", + extra=logging_context(), + ) + annotation = None # Send pure result data (no wrapper) result = { diff --git a/src/mavedb/routers/variants.py b/src/mavedb/routers/variants.py index c195f9030..bb76716c4 100644 --- a/src/mavedb/routers/variants.py +++ b/src/mavedb/routers/variants.py @@ -1,9 +1,11 @@ import itertools import logging import re +from typing import Any, List, Optional -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, Query from fastapi.exceptions import HTTPException +from fastapi.responses import StreamingResponse from sqlalchemy import select from sqlalchemy.exc import MultipleResultsFound from sqlalchemy.orm import Session, joinedload @@ -11,10 +13,15 @@ from mavedb import deps from mavedb.lib.authentication import get_current_user +from mavedb.lib.authorization import get_principal +from mavedb.lib.csv.namespaces import CSV_NAMESPACES_PARAM_DESCRIPTION, CsvNamespaceStr from mavedb.lib.logging import LoggedRoute from mavedb.lib.logging.context import logging_context, save_to_logging_context +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.types.authentication import UserData +from mavedb.lib.csv.variant import available_variant_csv_namespaces, get_variant_csv from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant @@ -25,6 +32,7 @@ PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX, ) +from mavedb.view_models.csv_namespace import AvailableCsvNamespace from mavedb.view_models.variant import ( ClingenAlleleIdVariantLookupResponse, ClingenAlleleIdVariantLookupsRequest, @@ -460,3 +468,139 @@ def get_variant(*, urn: str, db: Session = Depends(deps.get_db), user_data: User assert_permission(user_data, variant.score_set, Action.READ) return variant + + +@router.get( + "/variants/{urn}/csv-namespaces", + status_code=200, + response_model=list[AvailableCsvNamespace], + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="List the CSV column namespaces this variant has data for", +) +def get_variant_csv_namespaces( + *, + urn: str, + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), +) -> Any: + """ + List the CSV column namespaces this variant has data for, labeled and grouped for a picker. + + Widens over the variant's equivalent measurements the same way the CSV does, so a calibration + belonging to another score set that also measured this allele is offered here too. + + Parameters + __________ + urn : str + The URN of the variant to inspect. + db : Session + The database session to use. + user_data : Optional[UserData] + The user data of the current user. If None, no user-specific permissions are checked. + + Returns + _______ + list[AvailableCsvNamespace] + The namespaces with data, each with a human-readable label and group. + """ + save_to_logging_context({"requested_resource": urn, "resource_property": "csv-namespaces"}) + + variant = db.query(Variant).filter(Variant.urn == urn).one_or_none() + if not variant: + logger.info(msg="Could not fetch CSV namespaces; No such variant exists.", extra=logging_context()) + raise HTTPException(status_code=404, detail=f"variant with URN '{urn}' not found") + + assert_permission(user_data, variant.score_set, Action.READ) + + return available_variant_csv_namespaces( + db, + urn, + may_read_score_set=lambda score_set: has_permission(user_data, score_set, Action.READ).permitted, + viewer=principal.viewer_for(ScoreCalibrationViewer), + ) + + +@router.get( + "/variants/{urn}/csv", + status_code=200, + responses={ + 200: { + "content": {"text/csv": {}}, + "description": ( + "Variant data in CSV format, one row per measurement of the variant's allele. Columns" + " cover identity, mapped coordinates, the measured score, external annotations, and each" + " requested calibration's functional and ACMG interpretation." + ), + }, + **BASE_400_RESPONSE, + **ACCESS_CONTROL_ERROR_RESPONSES, + }, + summary="Get variant data in CSV format", +) +def get_variant_csv_data( + *, + urn: str, + namespaces: Optional[List[CsvNamespaceStr]] = Query(default=None, description=CSV_NAMESPACES_PARAM_DESCRIPTION), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), +) -> Any: + """ + Return tabular data for a single variant, identified by URN, in CSV format. + + Where the variant-level annotation endpoints return nested VA-Spec objects, this flattens the same + interpretation into columns a clinical information system can consume: ACMG criteria, evidence + strengths, and evidence outcome codes alongside the measurement they were derived from. + + A row is emitted for every current measurement of the variant's ClinGen allele, so a variant assayed + in several score sets yields several rows. The requested variant is always first. + + Parameters + __________ + urn : str + The URN of the variant to fetch. + namespaces : Optional[List[str]] + The groups of columns to include. When omitted, the response includes the fixed groups plus one + namespace per calibration eligible to annotate these measurements and the most recent ClinVar + release covering them. + db : Session + The database session to use. + user_data : Optional[UserData] + The user data of the current user. If None, no user-specific permissions are checked. + + Returns + _______ + Any + StreamingResponse containing the CSV data. + """ + save_to_logging_context({"requested_resource": urn, "resource_property": "csv", "namespaces": namespaces}) + + try: + variant = db.query(Variant).filter(Variant.urn == urn).one_or_none() + except MultipleResultsFound: + logger.info(msg="Could not fetch the requested variant; Multiple such variants exist.", extra=logging_context()) + raise HTTPException(status_code=500, detail=f"multiple variants with URN '{urn}' were found") + + if not variant: + logger.info(msg="Could not fetch the requested variant; No such variant exists.", extra=logging_context()) + raise HTTPException(status_code=404, detail=f"variant with URN '{urn}' not found") + + assert_permission(user_data, variant.score_set, Action.READ) + + # Only measurements the requester may read are emitted. The predicate runs against the score sets + # reached by the widening, keeping the permission check proportional to the result. + # A calibration's READ permission is stricter than its score set's, so it is asked separately: being + # able to read the measurement does not entitle a caller to a private calibration's interpretation. + csv_str = get_variant_csv( + db, + urn, + namespaces=namespaces, + may_read_score_set=lambda score_set: has_permission(user_data, score_set, Action.READ).permitted, + viewer=principal.viewer_for(ScoreCalibrationViewer), + ) + return StreamingResponse( + iter([csv_str]), + media_type="text/csv", + headers={"Content-Disposition": f'attachment; filename="{urn}.csv"'}, + ) diff --git a/src/mavedb/scripts/export_public_data.py b/src/mavedb/scripts/export_public_data.py index 3ce31a2ba..aaaafc831 100644 --- a/src/mavedb/scripts/export_public_data.py +++ b/src/mavedb/scripts/export_public_data.py @@ -26,9 +26,14 @@ from sqlalchemy.orm import Session, joinedload, lazyload from mavedb.lib.annotation.annotate import variant_highest_level_annotation +from mavedb.lib.csv.namespaces import CsvNamespace +from mavedb.lib.csv.score_set import ( + available_score_set_csv_namespaces, + get_score_set_variants_as_csv, +) from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer -from mavedb.lib.score_sets import get_current_mapped_variants_for_annotation, get_score_set_variants_as_csv +from mavedb.lib.score_sets import get_current_mapped_variants_for_annotation from mavedb.models.experiment import Experiment from mavedb.models.experiment_set import ExperimentSet from mavedb.models.license import License @@ -45,6 +50,39 @@ T = TypeVar("T") +def annotation_export_namespaces(db: Session, score_set: ScoreSet) -> list[str]: + """The namespaces the public annotations CSV should carry for this score set. + + Asks discovery what the score set actually has rather than naming groups by hand. The previous + hand-maintained list enumerated ClinVar releases one by one, so it emitted all-NA columns for releases + never ingested, needed a code change for every new release, and was fragile to schema changes. + + The archive carries everything MaveDB holds about the score set, so this takes what discovery found + and subtracts from it rather than opting groups in. + + In particular it does not filter on `selected_by_default`. That flag answers "what should a download + dialog open on", which is a question about attention rather than about what exists, and the reasons a + group opens unchecked are not interchangeable. An archive is about completeness, not about what a user + should be nudged to look at first. + + Subtractions: + + - Every score and count group, and the score set's own identity: scores and counts get their own + files, and the URN is in the filename, so repeating either would be noise. + """ + excluded = { + CsvNamespace.SCORES, + CsvNamespace.SCORES_CUSTOM, + CsvNamespace.COUNTS, + CsvNamespace.SCORE_SET, + } + return [ + entry.namespace + for entry in available_score_set_csv_namespaces(db, score_set) + if entry.namespace not in excluded + ] + + def flatmap(f: Callable[[S], Iterable[T]], items: Iterable[S]) -> Iterable[T]: return chain.from_iterable(map(f, items)) @@ -204,24 +242,7 @@ def export_public_data(db: Session): csv_str = get_score_set_variants_as_csv( db, score_set, - [ - "vep", - "gnomad", - "clingen", - "clinvar.2015_02", - "clinvar.2016_01", - "clinvar.2017_01", - "clinvar.2018_01", - "clinvar.2019_01", - "clinvar.2020_01", - "clinvar.2021_01", - "clinvar.2022_01", - "clinvar.2023_01", - "clinvar.2024_01", - "clinvar.2025_01", - "clinvar.2026_01", - ], - include_post_mapped_hgvs=True, + annotation_export_namespaces(db, score_set), namespaced=True, ) zipfile.writestr(f"csv/{csv_filename_base}.annotations.csv", csv_str) diff --git a/src/mavedb/scripts/resources/README.md b/src/mavedb/scripts/resources/README.md index 31ec4e26c..30d244a62 100644 --- a/src/mavedb/scripts/resources/README.md +++ b/src/mavedb/scripts/resources/README.md @@ -35,7 +35,7 @@ mavedb-dump.YYYYMMDDHHMMSS.zip ├── csv/ │ ├── {urn}.scores.csv # Variant effect scores (all score sets) │ ├── {urn}.counts.csv # Variant counts (score sets with count data only) -│ └── {urn}.annotations.csv # Variant annotations from VEP, gnomAD, and ClinGen +│ └── {urn}.annotations.csv # Variant annotations from VEP, gnomAD, ClinGen and ClinVar, plus score calibration interpretations │ # (score sets that have completed mapping only) ├── mapped/ │ └── {urn}.mapped-variants.json # Mapped variant data including VRS alleles and HGVS @@ -114,7 +114,9 @@ present for score sets that have count data. The count column names are listed i Variant annotation data from external databases, joined with post-mapped HGVS and VRS identifiers produced by the MaveDB variant mapping pipeline. **Only present for score sets that have completed -the MaveDB mapping pipeline.** Exact columns: +the MaveDB mapping pipeline.** + +Columns are grouped by a namespace prefix. The groups below always appear: | Column | Description | |--------|-------------| @@ -131,6 +133,39 @@ the MaveDB mapping pipeline.** Exact columns: | `gnomad.gnomad_af` | gnomAD v4.1 allele frequency | | `clingen.clingen_allele_id` | ClinGen Allele Registry CA identifier (e.g. `CA12345`) | +Two further groups vary by score set, because they exist only where MaveDB holds the underlying data. +Read the header rather than assuming a fixed column set. + +**ClinVar** — one pair of columns per ingested release, prefixed `clinvar.YEAR_MONTH`. A score set with +records from the January 2024 release carries `clinvar.2024_01.clinical_significance` and +`clinvar.2024_01.clinical_review_status`. + +This file carries **every** release MaveDB holds for the score set, not just the most recent one, so a +change in ClinVar's assessment over time can be read off a single file. + +**Score calibrations** — one group per calibration, prefixed `calibration.`, giving +that calibration's interpretation of each variant: + +| Column suffix | Description | +|---------------|-------------| +| `title` | Human-readable name of the calibration | +| `research_use_only` | Always `False` here; research-use-only calibrations are excluded from this dump | +| `functional_classification` | `normal`, `abnormal`, or `indeterminate` | +| `acmg_criterion` | ACMG 2015 criterion evaluated, e.g. `PS3` or `BS3` | +| `acmg_evidence_strength` | Strength the criterion was met at, e.g. `MODERATE`. `NA` when not met | +| `acmg_evidence_outcome_code` | ACMG evidence outcome code, e.g. `PS3_moderate`, `PS3` (strong), `BS3_not_met` | +| `pathogenicity_classification` | `PATHOGENIC`, `BENIGN`, or `UNCERTAIN_SIGNIFICANCE` | + +Every calibration MaveDB holds for the score set gets a group, including one that defines no score +ranges. Such a group carries `title` and +`research_use_only` with `NA` in every interpretation column: the calibration exists and was consulted, +and it has no classification to give. That is different from a calibration whose ranges simply do not +contain a particular variant, which reports `UNCERTAIN_SIGNIFICANCE` and `PS3_not_met`. + +`acmg_evidence_strength` uses MaveDB's own scale, which includes `MODERATE_PLUS` — an intermediate +strength that the GA4GH VA-Spec has no equivalent for. The same variant's record in `va/{urn}.va.ndjson` +therefore reports `moderate` where this file reports `MODERATE_PLUS`. + Variants that could not be mapped, or for which a specific annotation is unavailable, will have `NA` in the corresponding column. For multi-allelic variants (haplotypes), `mavedb.*` HGVS columns will be `NA` because a single combined HGVS string cannot currently be derived. This may be updated in diff --git a/src/mavedb/view_models/csv_namespace.py b/src/mavedb/view_models/csv_namespace.py new file mode 100644 index 000000000..e4048cf2e --- /dev/null +++ b/src/mavedb/view_models/csv_namespace.py @@ -0,0 +1,43 @@ +from typing import Optional + +from mavedb.lib.csv.namespaces import CsvNamespaceGroup +from mavedb.view_models import record_type_validator, set_record_type +from mavedb.view_models.base.base import BaseModel +from mavedb.view_models.score_set import ShorterScoreSet + + +class AvailableCsvNamespace(BaseModel): + """One CSV column namespace a record has data for, ready to be offered as a choice. + + Labels are served rather than derived client-side: only the server knows a calibration's title or a + ClinVar release date. + """ + + record_type: str = None # type: ignore + + namespace: str + """The value to pass back in the ``namespaces`` query parameter.""" + + label: str + """Human-readable name for a picker.""" + + group: CsvNamespaceGroup + """Which section of a picker this belongs in.""" + + score_set: Optional[ShorterScoreSet] = None + """The score set a calibration namespace belongs to; None for namespaces that apply to any. + + A picker should group calibrations by this when a response spans more than one score set. + """ + + selected_by_default: bool = True + """Whether a picker should open with this group checked. + + False for research-use-only calibrations and for calibrations with no ranges: both are offered, but + neither should be swept into a download unasked. + """ + + _record_type_factory = record_type_validator()(set_record_type) + + class Config: + from_attributes = True diff --git a/src/mavedb/view_models/score_set.py b/src/mavedb/view_models/score_set.py index 84c445eeb..ce7a0730a 100644 --- a/src/mavedb/view_models/score_set.py +++ b/src/mavedb/view_models/score_set.py @@ -244,16 +244,12 @@ def as_form(cls, **kwargs: Any) -> "ScoreSetUpdateAllOptional": # Define which fields need special JSON parsing json_fields = { "contributors": lambda data: [ContributorCreate.model_validate(c) for c in data] if data else None, - "primary_publication_identifiers": lambda data: [ - PublicationIdentifierCreate.model_validate(p) for p in data - ] - if data - else None, - "secondary_publication_identifiers": lambda data: [ - PublicationIdentifierCreate.model_validate(s) for s in data - ] - if data - else None, + "primary_publication_identifiers": lambda data: ( + [PublicationIdentifierCreate.model_validate(p) for p in data] if data else None + ), + "secondary_publication_identifiers": lambda data: ( + [PublicationIdentifierCreate.model_validate(s) for s in data] if data else None + ), "doi_identifiers": lambda data: [DoiIdentifierCreate.model_validate(d) for d in data] if data else None, "target_genes": lambda data: [TargetGeneCreate.model_validate(t) for t in data] if data else None, "extra_metadata": lambda data: data, @@ -326,7 +322,10 @@ def generate_primary_and_secondary_publications(cls, data: Any): class ShorterScoreSet(BaseModel): + """A score set's identity: enough to name it in a UI without rooting the display on its URN.""" + urn: str + title: str record_type: str = None # type: ignore _record_type_factory = record_type_validator()(set_record_type) diff --git a/tests/lib/annotation/test_flatten.py b/tests/lib/annotation/test_flatten.py new file mode 100644 index 000000000..21ecfa173 --- /dev/null +++ b/tests/lib/annotation/test_flatten.py @@ -0,0 +1,160 @@ +import pytest + +from mavedb.lib.annotation.flatten import FlatAnnotation, flatten_annotation +from mavedb.models.enums.acmg_criterion import ACMGCriterion +from mavedb.models.enums.functional_classification import FunctionalClassification as FunctionalClassificationOptions +from mavedb.models.enums.strength_of_evidence import StrengthOfEvidenceProvided +from tests.helpers.mocks.factories import ( + create_mock_acmg_classification, + create_mock_functional_classification, + create_mock_mapped_variant, + create_mock_score_calibration, + create_mock_score_set, +) + + +def _calibration_and_variant(functional_classifications, **calibration_kwargs): + """Build a mock mapped variant whose score set carries a single calibration with *functional_classifications*.""" + score_set = create_mock_score_set() + calibration = create_mock_score_calibration( + functional_classifications=functional_classifications, + score_set=score_set, + **calibration_kwargs, + ) + score_set.score_calibrations = [calibration] + return calibration, create_mock_mapped_variant(score_set=score_set) + + +def _pathogenicity_calibration_and_variant( + criterion=ACMGCriterion.PS3, + evidence_strength=StrengthOfEvidenceProvided.STRONG, + variant_in_range=True, + **calibration_kwargs, +): + classification = create_mock_functional_classification( + functional_classification=FunctionalClassificationOptions.abnormal, + label="Abnormal Range", + range_values=[0.7, 1.0], + acmg_classification=create_mock_acmg_classification( + criterion=criterion, + evidence_strength=evidence_strength, + ), + variant_in_range=variant_in_range, + ) + return _calibration_and_variant([classification], **calibration_kwargs) + + +class TestFlattenAnnotation: + """Tests that flatten_annotation projects a calibration's interpretation onto scalar fields.""" + + def test_pathogenicity_calibration_populates_all_fields(self): + calibration, mapped_variant = _pathogenicity_calibration_and_variant( + urn="urn:mavedb:calibration:1", title="Clinical Calibration" + ) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.functional_classification == "abnormal" + assert annotation.acmg_criterion == "PS3" + assert annotation.acmg_evidence_strength == "STRONG" + assert annotation.acmg_evidence_outcome_code == "PS3" + assert annotation.pathogenicity_classification == "PATHOGENIC" + assert annotation.calibration_urn == "urn:mavedb:calibration:1" + assert annotation.calibration_title == "Clinical Calibration" + + def test_benign_criterion_yields_benign_classification(self): + calibration, mapped_variant = _pathogenicity_calibration_and_variant(criterion=ACMGCriterion.BS3) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.acmg_criterion == "BS3" + assert annotation.pathogenicity_classification == "BENIGN" + + def test_functional_only_calibration_omits_acmg_fields(self): + classification = create_mock_functional_classification( + functional_classification=FunctionalClassificationOptions.normal, + label="Normal Range", + range_values=[-1.0, 0.3], + ) + calibration, mapped_variant = _calibration_and_variant( + [classification], urn="urn:mavedb:calibration:2", title="Functional Calibration" + ) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.functional_classification == "normal" + assert annotation.acmg_criterion is None + assert annotation.acmg_evidence_strength is None + assert annotation.acmg_evidence_outcome_code is None + assert annotation.pathogenicity_classification is None + assert annotation.calibration_urn == "urn:mavedb:calibration:2" + assert annotation.calibration_title == "Functional Calibration" + + def test_no_calibration_yields_empty_annotation(self): + mapped_variant = create_mock_mapped_variant() + + assert flatten_annotation(mapped_variant, None) == FlatAnnotation() + + def test_calibration_without_ranges_keeps_its_identity(self): + """Which calibration was consulted is known even when it can classify nothing. + + Reporting it is what distinguishes a calibration that defines no ranges from no calibration at + all, and the public dump carries the former. + """ + calibration, mapped_variant = _calibration_and_variant( + [], urn="urn:mavedb:calibration:3", title="Baseline Only" + ) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.calibration_urn == "urn:mavedb:calibration:3" + assert annotation.calibration_title == "Baseline Only" + assert annotation.research_use_only is False + assert annotation.functional_classification is None + assert annotation.acmg_criterion is None + assert annotation.acmg_evidence_strength is None + assert annotation.acmg_evidence_outcome_code is None + assert annotation.pathogenicity_classification is None + + def test_variant_outside_all_ranges_is_uncertain(self): + calibration, mapped_variant = _pathogenicity_calibration_and_variant(variant_in_range=False) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.functional_classification == "indeterminate" + assert annotation.acmg_criterion == "PS3" + assert annotation.acmg_evidence_strength is None + assert annotation.acmg_evidence_outcome_code == "PS3_not_met" + assert annotation.pathogenicity_classification == "UNCERTAIN_SIGNIFICANCE" + + @pytest.mark.parametrize( + "criterion,evidence_strength,expected_code", + [ + (ACMGCriterion.PS3, StrengthOfEvidenceProvided.STRONG, "PS3"), + (ACMGCriterion.PS3, StrengthOfEvidenceProvided.VERY_STRONG, "PS3_very_strong"), + (ACMGCriterion.PS3, StrengthOfEvidenceProvided.MODERATE, "PS3_moderate"), + (ACMGCriterion.PS3, StrengthOfEvidenceProvided.SUPPORTING, "PS3_supporting"), + (ACMGCriterion.BS3, StrengthOfEvidenceProvided.STRONG, "BS3"), + (ACMGCriterion.BS3, StrengthOfEvidenceProvided.SUPPORTING, "BS3_supporting"), + ], + ) + def test_evidence_outcome_code(self, criterion, evidence_strength, expected_code): + calibration, mapped_variant = _pathogenicity_calibration_and_variant( + criterion=criterion, evidence_strength=evidence_strength + ) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.acmg_evidence_outcome_code == expected_code + + def test_moderate_plus_is_preserved(self): + """The VA-Spec annotations must collapse M+ to moderate; a CSV has no such obligation.""" + calibration, mapped_variant = _pathogenicity_calibration_and_variant( + evidence_strength=StrengthOfEvidenceProvided.MODERATE_PLUS + ) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.acmg_evidence_strength == "MODERATE_PLUS" + assert annotation.acmg_evidence_outcome_code == "PS3_moderate_plus" + assert annotation.pathogenicity_classification == "PATHOGENIC" diff --git a/tests/lib/csv/__init__.py b/tests/lib/csv/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lib/csv/test_columns.py b/tests/lib/csv/test_columns.py new file mode 100644 index 000000000..719794bee --- /dev/null +++ b/tests/lib/csv/test_columns.py @@ -0,0 +1,622 @@ +import pytest + +from mavedb.lib.annotation.flatten import FlatAnnotation +from mavedb.lib.csv.columns import ( + _is_output_null, + assemble_csv_headers, + drop_unused_hgvs_columns, + plan_csv_columns, + rows_to_csv, + variant_to_csv_row, +) +from tests.helpers.constants import VALID_CALIBRATION_URN + +# --------------------------------------------------------------------------- +# MockVariant +# --------------------------------------------------------------------------- + + +class MockVariant: + """Lightweight mock for Variant used in variant_to_csv_row tests.""" + + def __init__(self, urn="urn:mavedb:00000001-a-1#1", hgvs_nt=None, hgvs_splice=None, hgvs_pro=None, data=None): + self.urn = urn + self.hgvs_nt = hgvs_nt + self.hgvs_splice = hgvs_splice + self.hgvs_pro = hgvs_pro + self.data = data + + +# --------------------------------------------------------------------------- +# TestVariantToCsvRowNullHandling +# --------------------------------------------------------------------------- + + +class TestVariantToCsvRowNullHandling: + """Tests that variant_to_csv_row represents missing data as na_rep, not 'None'.""" + + def test_score_data_with_none_value_uses_na_rep(self): + variant = MockVariant(data={"score_data": {"score": None}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "NA" + + def test_score_data_with_missing_key_uses_na_rep(self): + variant = MockVariant(data={"score_data": {}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "NA" + + def test_score_data_with_no_score_data_key_uses_na_rep(self): + variant = MockVariant(data={}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "NA" + + def test_score_data_with_no_data_uses_na_rep(self): + variant = MockVariant(data=None) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "NA" + + def test_count_data_with_none_value_uses_na_rep(self): + variant = MockVariant(data={"count_data": {"count1": None}}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "NA" + + def test_count_data_with_missing_key_uses_na_rep(self): + variant = MockVariant(data={"count_data": {}}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "NA" + + def test_count_data_with_no_count_data_key_uses_na_rep(self): + variant = MockVariant(data={}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "NA" + + def test_count_data_with_no_data_uses_na_rep(self): + variant = MockVariant(data=None) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "NA" + + def test_score_data_with_valid_value_preserved(self): + variant = MockVariant(data={"score_data": {"score": 1.5}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "1.5" + + def test_count_data_with_valid_value_preserved(self): + variant = MockVariant(data={"count_data": {"count1": 42}}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "42" + + def test_score_data_with_custom_na_rep(self): + variant = MockVariant(data={"score_data": {"score": None}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns, na_rep="N/A") + + assert row["score"] == "N/A" + + def test_namespaced_score_data_with_none_value_uses_na_rep(self): + variant = MockVariant(data={"score_data": {"score": None}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns, namespaced=True) + + assert row["scores.score"] == "NA" + + def test_namespaced_count_data_with_none_value_uses_na_rep(self): + variant = MockVariant(data={"count_data": {"count1": None}}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns, namespaced=True) + + assert row["counts.count1"] == "NA" + + def test_core_columns_with_none_hgvs_uses_na_rep(self): + variant = MockVariant(hgvs_nt=None, hgvs_pro=None, hgvs_splice=None, urn="urn:mavedb:00000001-a-1#1") + columns = {"core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"]} + + row = variant_to_csv_row(variant, columns) + + assert row["hgvs_nt"] == "NA" + assert row["hgvs_pro"] == "NA" + assert row["hgvs_splice"] == "NA" + assert row["accession"] == "urn:mavedb:00000001-a-1#1" + + def test_mixed_columns_with_missing_data(self): + variant = MockVariant( + hgvs_nt="g.1A>G", + hgvs_pro="p.Met1Val", + data={"score_data": {"score": None, "se": 0.1}, "count_data": {"count1": None, "count2": 5}}, + ) + columns = { + "core": ["hgvs_nt", "hgvs_pro"], + "scores": ["score", "se"], + "counts": ["count1", "count2"], + } + + row = variant_to_csv_row(variant, columns) + + assert row["hgvs_nt"] == "g.1A>G" + assert row["hgvs_pro"] == "p.Met1Val" + assert row["score"] == "NA" + assert row["se"] == "0.1" + assert row["count1"] == "NA" + assert row["count2"] == "5" + + +# --------------------------------------------------------------------------- +# TestVariantToCsvRowUnrecognizedKey +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.parametrize( + "namespace, columns", + [ + ("core", {"core": ["bogus_col"]}), + ("mavedb", {"mavedb": ["bogus_col"]}), + ("vep", {"vep": ["bogus_col"]}), + ("gnomad", {"gnomad": ["bogus_col"]}), + ("clingen", {"clingen": ["bogus_col"]}), + ("clinvar.2024_01", {"clinvar.2024_01": ["bogus_col"]}), + ("score_set", {"score_set": ["bogus_col"]}), + ("relationship", {"relationship": ["bogus_col"]}), + ("calibration", {f"calibration.{VALID_CALIBRATION_URN}": ["bogus_col"]}), + ], +) +def test_unrecognized_column_key_raises(namespace, columns): + variant = MockVariant() + with pytest.raises(ValueError, match="unrecognized .* column: bogus_col"): + variant_to_csv_row(variant, columns) + + +# --------------------------------------------------------------------------- +# TestCalibrationScoreSetAndRelationshipNamespaces +# --------------------------------------------------------------------------- + + +CALIBRATION_NS = f"calibration.{VALID_CALIBRATION_URN}" + +CALIBRATION_COLUMNS = [ + "title", + "functional_classification", + "acmg_criterion", + "acmg_evidence_strength", + "acmg_evidence_outcome_code", + "pathogenicity_classification", +] + + +class MockTargetGene: + def __init__(self, name): + self.name = name + + +class MockScoreSetForContext: + def __init__(self, urn="urn:mavedb:00000001-a-1", target_gene_names=()): + self.urn = urn + self.target_genes = [MockTargetGene(name) for name in target_gene_names] + + +class MockVariantWithScoreSet(MockVariant): + def __init__(self, score_set=None, **kwargs): + super().__init__(**kwargs) + self.score_set = score_set + + +class TestCalibrationNamespace: + """Tests that a calibration namespace renders a FlatAnnotation, keyed by the calibration's URN.""" + + def test_populated_annotation(self): + annotation = FlatAnnotation( + functional_classification="abnormal", + acmg_criterion="PS3", + acmg_evidence_strength="MODERATE", + acmg_evidence_outcome_code="PS3_moderate", + pathogenicity_classification="PATHOGENIC", + calibration_urn=VALID_CALIBRATION_URN, + calibration_title="Clinical Calibration", + ) + + row = variant_to_csv_row( + MockVariant(), + {CALIBRATION_NS: CALIBRATION_COLUMNS}, + annotations_by_ns={CALIBRATION_NS: annotation}, + ) + + assert row[f"{CALIBRATION_NS}.title"] == "Clinical Calibration" + assert row[f"{CALIBRATION_NS}.functional_classification"] == "abnormal" + assert row[f"{CALIBRATION_NS}.acmg_criterion"] == "PS3" + assert row[f"{CALIBRATION_NS}.acmg_evidence_strength"] == "MODERATE" + assert row[f"{CALIBRATION_NS}.acmg_evidence_outcome_code"] == "PS3_moderate" + assert row[f"{CALIBRATION_NS}.pathogenicity_classification"] == "PATHOGENIC" + + def test_calibration_columns_are_always_namespaced(self): + """The URN in the header is what disambiguates, so it is kept even for un-namespaced output.""" + annotation = FlatAnnotation(acmg_criterion="PS3") + + row = variant_to_csv_row( + MockVariant(), + {CALIBRATION_NS: ["acmg_criterion"]}, + annotations_by_ns={CALIBRATION_NS: annotation}, + namespaced=False, + ) + + assert row == {f"{CALIBRATION_NS}.acmg_criterion": "PS3"} + + def test_no_annotation_uses_na_rep(self): + row = variant_to_csv_row(MockVariant(), {CALIBRATION_NS: CALIBRATION_COLUMNS}) + + assert all(row[f"{CALIBRATION_NS}.{column}"] == "NA" for column in CALIBRATION_COLUMNS) + + def test_annotation_absent_for_this_namespace_uses_na_rep(self): + other_ns = "calibration.urn:mavedb:calibration-00000000-0000-0000-0000-000000000000" + + row = variant_to_csv_row( + MockVariant(), + {CALIBRATION_NS: ["acmg_criterion"]}, + annotations_by_ns={other_ns: FlatAnnotation(acmg_criterion="PS3")}, + ) + + assert row[f"{CALIBRATION_NS}.acmg_criterion"] == "NA" + + def test_multiple_calibration_namespaces_are_independent(self): + first_ns = CALIBRATION_NS + second_ns = "calibration.urn:mavedb:calibration-00000000-0000-0000-0000-000000000000" + + row = variant_to_csv_row( + MockVariant(), + {first_ns: ["acmg_criterion"], second_ns: ["acmg_criterion"]}, + annotations_by_ns={ + first_ns: FlatAnnotation(acmg_criterion="PS3"), + second_ns: FlatAnnotation(acmg_criterion="BS3"), + }, + ) + + assert row[f"{first_ns}.acmg_criterion"] == "PS3" + assert row[f"{second_ns}.acmg_criterion"] == "BS3" + + +class TestScoreSetNamespace: + """Tests that the score_set namespace reports the row's score set and target genes.""" + + def test_populated_score_set(self): + variant = MockVariantWithScoreSet(score_set=MockScoreSetForContext(target_gene_names=["BRCA1", "BRCA2"])) + columns = {"score_set": ["score_set_urn", "target_gene"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score_set_urn"] == "urn:mavedb:00000001-a-1" + assert row["target_gene"] == "BRCA1; BRCA2" + + def test_empty_collections_use_na_rep(self): + variant = MockVariantWithScoreSet(score_set=MockScoreSetForContext()) + + row = variant_to_csv_row(variant, {"score_set": ["target_gene"]}) + + assert row["target_gene"] == "NA" + + def test_publication_identifiers_is_not_a_column(self): + """Dropped: it repeats on every row of a score set and score_set_urn already resolves to it.""" + variant = MockVariantWithScoreSet(score_set=MockScoreSetForContext()) + + with pytest.raises(ValueError, match="unrecognized score_set column: publication_identifiers"): + variant_to_csv_row(variant, {"score_set": ["publication_identifiers"]}) + + +class TestRelationshipNamespace: + """Tests that the relationship namespace reports the caller-supplied match type.""" + + def test_populated_match_type(self): + row = variant_to_csv_row(MockVariant(), {"relationship": ["match_type"]}, match_type="exact") + + assert row["match_type"] == "exact" + + def test_missing_match_type_uses_na_rep(self): + row = variant_to_csv_row(MockVariant(), {"relationship": ["match_type"]}) + + assert row["match_type"] == "NA" + + +# --------------------------------------------------------------------------- +# TestRowsToCsv +# --------------------------------------------------------------------------- + + +class TestRowsToCsv: + def test_header_only_when_no_rows(self): + assert rows_to_csv([], ["a", "b"]).splitlines() == ["a,b"] + + def test_writes_rows_in_column_order(self): + rows = [{"b": "2", "a": "1"}, {"a": "3", "b": "4"}] + + assert rows_to_csv(rows, ["a", "b"]).splitlines() == ["a,b", "1,2", "3,4"] + + def test_quotes_values_containing_commas(self): + assert rows_to_csv([{"a": "x,y"}], ["a"]).splitlines()[1] == '"x,y"' + + +# --------------------------------------------------------------------------- +# TestPlanCsvColumns +# --------------------------------------------------------------------------- + + +SAMPLE_DATASET_COLUMNS = { + "score_columns": ["score", "se", "epsilon"], + "count_columns": ["count1", "count2"], +} + + +@pytest.mark.unit +@pytest.mark.parametrize( + "namespaces, expected_ns_keys, expected_columns, expected_clinvar", + [ + # `scores` is the one column dataframe validation mandates, nothing more. + (["scores"], {"core", "scores"}, {"scores": ["score"]}, {}), + # The investigator's remaining score columns are their own request token. + (["scores_custom"], {"core", "scores_custom"}, {"scores_custom": ["se", "epsilon"]}, {}), + # Asking for both reproduces the whole score group, `score` first. + ( + ["scores", "scores_custom"], + {"core", "scores", "scores_custom"}, + {"scores": ["score"], "scores_custom": ["se", "epsilon"]}, + {}, + ), + # Counts have no required column, so they are always taken in full. + (["counts"], {"core", "counts"}, {"counts": ["count1", "count2"]}, {}), + ( + ["scores", "counts"], + {"core", "scores", "counts"}, + {"scores": ["score"], "counts": ["count1", "count2"]}, + {}, + ), + (["vep"], {"core", "vep"}, {"vep": ["vep_functional_consequence"]}, {}), + (["gnomad"], {"core", "gnomad"}, {"gnomad": ["gnomad_af"]}, {}), + (["clingen"], {"core", "clingen"}, {"clingen": ["clingen_allele_id"]}, {}), + (["scores", "mavedb"], {"core", "scores", "mavedb"}, {"scores": ["score"]}, {}), + (["clinvar.2024_01"], {"core", "clinvar.2024_01"}, {}, {"clinvar.2024_01": "01_2024"}), + ( + ["clinvar.2024_01", "clinvar.2025_06"], + {"core", "clinvar.2024_01", "clinvar.2025_06"}, + {}, + {"clinvar.2024_01": "01_2024", "clinvar.2025_06": "06_2025"}, + ), + # A namespace requested twice is planned once, or it would emit its columns twice. + (["scores", "scores"], {"core", "scores"}, {"scores": ["score"]}, {}), + ], +) +def test_plan_csv_columns(namespaces, expected_ns_keys, expected_columns, expected_clinvar): + plan = plan_csv_columns(SAMPLE_DATASET_COLUMNS, namespaces) + + assert set(plan.namespaced_columns.keys()) == expected_ns_keys + assert plan.clinvar_namespaces == expected_clinvar + for namespace, columns in expected_columns.items(): + assert plan.namespaced_columns[namespace] == columns + + assert plan.namespaced_columns["core"] == ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"] + + for ns in expected_clinvar: + assert plan.namespaced_columns[ns] == ["clinical_significance", "clinical_review_status"] + + +def test_plan_csv_columns_reference_hgvs_namespace_populates_columns(): + plan = plan_csv_columns(SAMPLE_DATASET_COLUMNS, ["scores", "mavedb"]) + assert plan.namespaced_columns["mavedb"] == [ + "post_mapped_hgvs_g", + "post_mapped_hgvs_p", + "post_mapped_hgvs_c", + "post_mapped_hgvs_at_assay_level", + "post_mapped_vrs_digest", + ] + + +# --------------------------------------------------------------------------- +# TestAssembleCsvHeaders +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.parametrize( + "namespaced_columns, namespaced, expected", + [ + # Unnamespaced: flat column names + ( + {"core": ["accession", "hgvs_nt"], "scores": ["score", "se"]}, + False, + ["accession", "hgvs_nt", "score", "se"], + ), + # Namespaced: scores get prefix, core does not + ( + {"core": ["accession", "hgvs_nt"], "scores": ["score"]}, + True, + ["accession", "hgvs_nt", "scores.score"], + ), + # mavedb namespace always gets prefix when namespaced + ( + {"core": ["accession"], "mavedb": ["post_mapped_hgvs_g"]}, + True, + ["accession", "mavedb.post_mapped_hgvs_g"], + ), + # ClinVar namespaces always get prefix regardless of namespaced flag + ( + {"core": ["accession"], "clinvar.2024_01": ["clinical_significance"]}, + False, + ["accession", "clinvar.2024_01.clinical_significance"], + ), + # Mixed: respects insertion order + ( + { + "core": ["accession"], + "mavedb": [], + "scores": ["score"], + "clinvar.2024_01": ["clinical_significance"], + }, + True, + ["accession", "scores.score", "clinvar.2024_01.clinical_significance"], + ), + # Empty mavedb namespace when not namespaced produces nothing + ( + {"core": ["hgvs_nt"], "mavedb": []}, + False, + ["hgvs_nt"], + ), + ], +) +def test_assemble_csv_headers(namespaced_columns, namespaced, expected): + assert assemble_csv_headers(namespaced_columns, namespaced) == expected + + +# --------------------------------------------------------------------------- +# TestDropNaColumns +# --------------------------------------------------------------------------- + + +class TestDropNaColumns: + def test_removes_all_na_hgvs_column(self): + rows = [ + {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "p.Met1Val"}, + {"hgvs_nt": "g.2C>T", "hgvs_splice": "NA", "hgvs_pro": "p.Ala2Gly"}, + ] + columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + + new_rows, new_cols = drop_unused_hgvs_columns(rows, columns) + + assert "hgvs_splice" not in new_cols + assert "hgvs_nt" in new_cols + assert "hgvs_pro" in new_cols + for row in new_rows: + assert "hgvs_splice" not in row + + def test_keeps_column_with_some_values(self): + rows = [ + {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "p.Met1Val"}, + {"hgvs_nt": "g.2C>T", "hgvs_splice": "c.1A>G", "hgvs_pro": "p.Ala2Gly"}, + ] + columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + + new_rows, new_cols = drop_unused_hgvs_columns(rows, columns) + + assert new_cols == ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + + def test_does_not_touch_non_hgvs_columns(self): + rows = [ + {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "NA", "score": "NA"}, + ] + columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro", "score"] + + new_rows, new_cols = drop_unused_hgvs_columns(rows, columns) + + assert "score" in new_cols + assert "hgvs_splice" not in new_cols + + def test_empty_rows_does_not_crash(self): + rows = [] + columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + + new_rows, new_cols = drop_unused_hgvs_columns(rows, columns) + + assert new_rows == [] + assert new_cols == [] + + +def test_plan_csv_columns_omits_reference_hgvs_when_not_requested(): + """It used to be a boolean flag, so the key was always present even when empty.""" + plan = plan_csv_columns(SAMPLE_DATASET_COLUMNS, ["scores"]) + + assert "mavedb" not in plan.namespaced_columns + + +# --------------------------------------------------------------------------- +# TestIsOutputNull +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.parametrize( + "value, expected", + [ + (None, True), + ("", True), + (" ", True), + ("NA", True), + ("na", True), + ("None", True), + ("none", True), + ("NaN", True), + ("nan", True), + ("null", True), + ("NULL", True), + ("nil", True), + ("N/A", True), + ("undefined", True), + ("1.5", False), + ("0", False), + ("hello", False), + ("p.Met1Val", False), + ], +) +def test_is_output_null(value, expected): + assert _is_output_null(value) is expected + + +@pytest.mark.unit +class TestAssembleCsvHeadersRejectsCollisions: + """Un-namespaced output strips the prefix that keeps two namespaces' columns apart. + + The endpoints that ask for un-namespaced output request one namespace each; this holds them to it + rather than letting a future caller discover the problem from a CSV with a column written twice. + """ + + def test_un_namespaced_collision_raises(self): + with pytest.raises(ValueError, match="duplicate columns"): + assemble_csv_headers({"scores": ["score"], "counts": ["score"]}, namespaced=False) + + def test_the_error_names_the_offending_column_and_namespaces(self): + with pytest.raises(ValueError) as excinfo: + assemble_csv_headers({"scores": ["shared"], "counts": ["shared"]}, namespaced=False) + + assert "shared" in str(excinfo.value) + assert "scores" in str(excinfo.value) and "counts" in str(excinfo.value) + + def test_namespacing_resolves_what_would_otherwise_collide(self): + headers = assemble_csv_headers({"scores": ["shared"], "counts": ["shared"]}, namespaced=True) + + assert headers == ["scores.shared", "counts.shared"] + + def test_scores_and_custom_scores_share_a_prefix_without_colliding(self): + """They emit under one prefix by design, so the guard must not fire on disjoint column sets.""" + headers = assemble_csv_headers({"scores": ["score"], "scores_custom": ["se"]}, namespaced=True) + + assert headers == ["scores.score", "scores.se"] + + def test_a_namespace_sharing_a_prefix_still_collides_on_a_repeated_column(self): + with pytest.raises(ValueError, match="duplicate columns"): + assemble_csv_headers({"scores": ["score"], "scores_custom": ["score"]}, namespaced=True) diff --git a/tests/lib/csv/test_entries.py b/tests/lib/csv/test_entries.py new file mode 100644 index 000000000..7a225c4ef --- /dev/null +++ b/tests/lib/csv/test_entries.py @@ -0,0 +1,64 @@ +import pytest + +from mavedb.lib.csv.entries import clinvar_namespace_entries +from mavedb.lib.csv.namespaces import CsvNamespaceGroup + +pytestmark = pytest.mark.unit + + +# --------------------------------------------------------------------------- +# ClinVar release entries +# +# These go through `clinvar_namespace_entries` rather than asserting how the namespace strings compare, +# because the ordering is what picks the default selection. A test of the comparison alone would keep +# passing if the call site stopped using it. +# --------------------------------------------------------------------------- + + +class TestClinvarNamespaceEntries: + def test_entries_are_ordered_newest_release_first(self): + entries = clinvar_namespace_entries(["clinvar.2024_02", "clinvar.2025_10", "clinvar.2024_11"]) + + assert [entry.namespace for entry in entries] == [ + "clinvar.2025_10", + "clinvar.2024_11", + "clinvar.2024_02", + ] + + def test_only_the_newest_release_is_selected_by_default(self): + entries = clinvar_namespace_entries(["clinvar.2024_02", "clinvar.2025_10", "clinvar.2024_11"]) + + assert [entry.selected_by_default for entry in entries] == [True, False, False] + + def test_ordering_survives_uneven_year_widths(self): + """The decay case: as plain strings "clinvar.999_12" sorts above every four-digit year. + + Ordering by the namespace string would put the malformed release first and hand it the default + selection, silently changing which ClinVar call a picker opens with. + """ + entries = clinvar_namespace_entries(["clinvar.999_12", "clinvar.2025_01"]) + + assert [entry.namespace for entry in entries] == ["clinvar.2025_01", "clinvar.999_12"] + assert entries[0].selected_by_default is True + assert entries[1].selected_by_default is False + + def test_duplicate_releases_are_collapsed(self): + entries = clinvar_namespace_entries(["clinvar.2025_01", "clinvar.2025_01"]) + + assert [entry.namespace for entry in entries] == ["clinvar.2025_01"] + + def test_unlabelable_namespaces_are_dropped_without_taking_the_default(self): + """An entry that cannot be labelled must not consume the one default slot on its way out.""" + entries = clinvar_namespace_entries(["clinvar.2024_13", "clinvar.2025_01"]) + + assert [entry.namespace for entry in entries] == ["clinvar.2025_01"] + assert entries[0].selected_by_default is True + + def test_entries_are_grouped_as_annotation(self): + entries = clinvar_namespace_entries(["clinvar.2025_01"]) + + assert entries[0].group is CsvNamespaceGroup.ANNOTATION + assert entries[0].label == "ClinVar significance (January 2025)" + + def test_no_releases_yields_no_entries(self): + assert clinvar_namespace_entries([]) == [] diff --git a/tests/lib/csv/test_namespaces.py b/tests/lib/csv/test_namespaces.py new file mode 100644 index 000000000..a5166ebf4 --- /dev/null +++ b/tests/lib/csv/test_namespaces.py @@ -0,0 +1,242 @@ +import pytest + +from pydantic import TypeAdapter, ValidationError + +from mavedb.lib.csv.namespaces import ( + CSV_NAMESPACE_ERROR_MESSAGE, + STATIC_CSV_NAMESPACES, + CsvNamespace, + CsvNamespaceStr, + calibration_namespace_for_urn, + clinvar_namespace_for_db_version, + clinvar_namespace_sort_key, + is_valid_csv_namespace, + parse_calibration_namespace, + parse_clinvar_db_version, + parse_clinvar_namespace, +) +from tests.helpers.constants import VALID_CALIBRATION_URN + +pytestmark = pytest.mark.unit + + +# --------------------------------------------------------------------------- +# ClinVar namespaces +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "ns, expected", + [ + ("clinvar.2024_01", "01_2024"), + ("clinvar.2015_12", "12_2015"), + ("clinvar.2026_06", "06_2026"), + ("clinvar.2024_00", None), + ("clinvar.2024_13", None), + ("scores", None), + ("clinvar", None), + ("clinvar.2024_01.extra", None), + ("", None), + ], +) +def test_parse_clinvar_namespace(ns, expected): + assert parse_clinvar_namespace(ns) == expected + + +@pytest.mark.parametrize( + "db_version, expected", + [ + ("01_2024", (2024, 1)), + ("12_2015", (2015, 12)), + ("2024", None), + ("aa_bbbb", None), + ("", None), + ], +) +def test_parse_clinvar_db_version(db_version, expected): + assert parse_clinvar_db_version(db_version) == expected + + +@pytest.mark.parametrize( + "db_version, expected", + [ + ("01_2024", "clinvar.2024_01"), + ("11_2024", "clinvar.2024_11"), + ("2_2025", "clinvar.2025_02"), + ("nonsense", None), + ], +) +def test_clinvar_namespace_for_db_version(db_version, expected): + assert clinvar_namespace_for_db_version(db_version) == expected + + +def test_clinvar_namespace_round_trips_through_db_version(): + assert clinvar_namespace_for_db_version(parse_clinvar_namespace("clinvar.2024_01")) == "clinvar.2024_01" + + +@pytest.mark.parametrize( + "ns, expected", + [ + ("clinvar.2024_01", (2024, 1)), + ("clinvar.2025_10", (2025, 10)), + # Not a release namespace at all: must sort below every real one rather than raise. + ("scores", (-1, -1)), + ("calibration.urn:mavedb:calibration-abc", (-1, -1)), + ("clinvar.2024_13", (-1, -1)), + ], +) +def test_clinvar_namespace_sort_key(ns, expected): + assert clinvar_namespace_sort_key(ns) == expected + + +def test_clinvar_namespaces_sort_chronologically(): + namespaces = ["clinvar.2024_11", "clinvar.2025_02", "clinvar.2024_02", "clinvar.2025_10"] + + assert max(namespaces, key=clinvar_namespace_sort_key) == "clinvar.2025_10" + assert sorted(namespaces, key=clinvar_namespace_sort_key) == [ + "clinvar.2024_02", + "clinvar.2024_11", + "clinvar.2025_02", + "clinvar.2025_10", + ] + + +def test_sort_key_beats_string_ordering_on_uneven_year_widths(): + """The year group in CLINVAR_NS_PATTERN is unpadded, so string ordering is not chronological. + + This is the decay this key exists to prevent: as plain strings "clinvar.999_12" sorts above + "clinvar.2025_01", which would make a malformed release look like the newest one and hand it the + picker's default selection. + """ + namespaces = ["clinvar.2025_01", "clinvar.999_12"] + + assert max(namespaces) == "clinvar.999_12" + assert max(namespaces, key=clinvar_namespace_sort_key) == "clinvar.2025_01" + + +def test_undatable_namespace_never_sorts_newest(): + assert max(["clinvar.2024_01", "scores"], key=clinvar_namespace_sort_key) == "clinvar.2024_01" + + +# --------------------------------------------------------------------------- +# Calibration namespaces +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "ns, expected", + [ + (f"calibration.{VALID_CALIBRATION_URN}", VALID_CALIBRATION_URN), + ( + "calibration.urn:mavedb:calibration-00000000-0000-0000-0000-000000000000", + "urn:mavedb:calibration-00000000-0000-0000-0000-000000000000", + ), + ("calibration.urn:mavedb:00000001-a-1", None), + ("calibration.not-a-urn", None), + ("calibration.", None), + ("calibration", None), + ("scores", None), + ("", None), + ], +) +def test_parse_calibration_namespace(ns, expected): + assert parse_calibration_namespace(ns) == expected + + +def test_calibration_namespace_round_trips(): + namespace = calibration_namespace_for_urn(VALID_CALIBRATION_URN) + + assert namespace == f"calibration.{VALID_CALIBRATION_URN}" + assert parse_calibration_namespace(namespace) == VALID_CALIBRATION_URN + + +@pytest.mark.parametrize( + "urn", + [ + "urn:mavedb:collection-79471b5b-2dbd-4a96-833c-c33023862437", + "urn:mavedb:00000001-a-1", + "urn:mavedb:calibration-short", + ], +) +def test_non_calibration_urns_do_not_form_valid_namespaces(urn): + assert parse_calibration_namespace(f"calibration.{urn}") is None + + +# --------------------------------------------------------------------------- +# Validation +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("ns", STATIC_CSV_NAMESPACES) +def test_static_namespaces_are_valid(ns): + assert is_valid_csv_namespace(ns) + + +@pytest.mark.parametrize( + "ns", + [ + "clinvar.2024_01", + f"calibration.{VALID_CALIBRATION_URN}", + ], +) +def test_parameterized_namespaces_are_valid(ns): + assert is_valid_csv_namespace(ns) + + +@pytest.mark.parametrize( + "ns", + [ + "bogus", + "clinvar", + "clinvar.2024_13", + "calibration", + "calibration.nope", + "SCORES", + "", + ], +) +def test_invalid_namespaces_are_rejected(ns): + assert not is_valid_csv_namespace(ns) + + +def test_static_namespaces_are_the_enum_values(): + """The tuple and the enum must not drift; the tuple feeds the published JSON schema.""" + assert STATIC_CSV_NAMESPACES == tuple(ns.value for ns in CsvNamespace) + + +def test_enum_members_are_usable_as_plain_strings(): + """`plan_csv_columns` keys its column dict by these, so they must behave as their values.""" + assert CsvNamespace.SCORE_SET == "score_set" + assert f"{CsvNamespace.SCORE_SET}" == "score_set" + assert {"score_set": 1}[CsvNamespace.SCORE_SET] == 1 + + +# --------------------------------------------------------------------------- +# CsvNamespaceStr — the validated query-parameter type +# --------------------------------------------------------------------------- + + +_ADAPTER = TypeAdapter(CsvNamespaceStr) + + +@pytest.mark.parametrize( + "ns", + list(STATIC_CSV_NAMESPACES) + ["clinvar.2024_01", f"calibration.{VALID_CALIBRATION_URN}"], +) +def test_validated_type_accepts_valid_namespaces(ns): + assert _ADAPTER.validate_python(ns) == ns + + +@pytest.mark.parametrize("ns", ["bogus", "clinvar", "clinvar.2024_13", "calibration", "calibration.nope", ""]) +def test_validated_type_rejects_invalid_namespaces(ns): + with pytest.raises(ValidationError) as exc_info: + _ADAPTER.validate_python(ns) + + assert CSV_NAMESPACE_ERROR_MESSAGE in str(exc_info.value) + + +def test_error_message_names_the_whole_vocabulary(): + for ns in STATIC_CSV_NAMESPACES: + assert f'"{ns}"' in CSV_NAMESPACE_ERROR_MESSAGE + assert "clinvar.YEAR_MONTH" in CSV_NAMESPACE_ERROR_MESSAGE + assert "calibration." in CSV_NAMESPACE_ERROR_MESSAGE diff --git a/tests/lib/csv/test_specs.py b/tests/lib/csv/test_specs.py new file mode 100644 index 000000000..24ec510ed --- /dev/null +++ b/tests/lib/csv/test_specs.py @@ -0,0 +1,113 @@ +"""The descriptors are what keep planning, row assembly and fetching from drifting apart.""" + +import pytest + +from mavedb.lib.csv.columns import variant_to_csv_row +from mavedb.lib.csv.namespaces import CsvNamespace +from mavedb.lib.csv.specs import CORE_NAMESPACE, RowSource, namespace_spec +from tests.helpers.constants import VALID_CALIBRATION_URN + +CALIBRATION_NS = f"calibration.{VALID_CALIBRATION_URN}" + +SAMPLE_DATASET_COLUMNS = { + "score_columns": ["score", "se", "epsilon"], + "count_columns": ["count1", "count2"], +} + +EVERY_NAMESPACE = [CORE_NAMESPACE, *CsvNamespace, "clinvar.2024_01", CALIBRATION_NS] + + +class _TargetGene: + def __init__(self, name): + self.name = name + + +class _ScoreSet: + urn = "urn:mavedb:00000001-a-1" + target_genes = [_TargetGene("BRCA1")] + + +class _Variant: + """Enough of a Variant for every namespace's resolvers to run against.""" + + urn = "urn:mavedb:00000001-a-1#1" + hgvs_nt = "c.1A>G" + hgvs_splice = None + hgvs_pro = "p.Met1Val" + data = {"score_data": {"score": 1.0}, "count_data": {"count1": 2}} + score_set = _ScoreSet() + + +# --------------------------------------------------------------------------- +# TestNamespaceSpecs +# --------------------------------------------------------------------------- + + +class TestNamespaceSpecs: + def test_every_static_namespace_has_a_spec(self): + """A namespace in the published vocabulary with no descriptor would silently produce no columns.""" + for namespace in CsvNamespace: + assert namespace_spec(namespace) is not None, namespace + + def test_parameterized_namespaces_resolve_to_a_spec(self): + assert namespace_spec("clinvar.2024_01") is not None + assert namespace_spec(CALIBRATION_NS) is not None + assert namespace_spec("not_a_namespace") is None + + @pytest.mark.parametrize("namespace", [ns for ns in CsvNamespace] + ["clinvar.2024_01"]) + def test_every_declared_column_can_be_resolved(self, namespace): + """A column with no resolver raises at row-assembly time, one row into a download.""" + spec = namespace_spec(namespace) + assert spec is not None + for column_key in spec.columns(SAMPLE_DATASET_COLUMNS): + assert spec.resolver(column_key) is not None, f"{namespace}.{column_key}" + + def test_a_namespace_reading_through_a_relationship_declares_the_fetch_it_needs(self): + """Otherwise the fetch layer would not eager-load it and every row would pay a query.""" + for namespace in (CsvNamespace.REFERENCE_HGVS, CsvNamespace.VEP, CsvNamespace.CLINGEN): + assert namespace_spec(namespace).needs_mappings, namespace + + gnomad = namespace_spec(CsvNamespace.GNOMAD) + assert gnomad.needs_gnomad and gnomad.needs_mappings + + assert namespace_spec(CsvNamespace.SCORE_SET).needs_score_set + calibration = namespace_spec(CALIBRATION_NS) + assert calibration.needs_mappings and calibration.needs_score_set + + def test_resolvers_report_missing_data_rather_than_raising(self): + """Every source is optional on some row: an unmapped variant, a release with no record for it.""" + for namespace in CsvNamespace: + spec = namespace_spec(namespace) + if spec.source in (RowSource.VARIANT, RowSource.MATCH_TYPE): + continue + for column_key in spec.columns(SAMPLE_DATASET_COLUMNS): + assert spec.resolver(column_key)(None) is None, f"{namespace}.{column_key}" + + +# --------------------------------------------------------------------------- +# TestRowSourceDispatch +# +# A spec names its source; `variant_to_csv_row` is what turns that name into the datum the resolvers are +# called with. That mapping is the one part of the descriptor contract the tests above cannot see, and a +# source with no entry in it raises KeyError at row-assembly time — one row into a download, which is the +# failure mode these descriptors exist to prevent. +# --------------------------------------------------------------------------- + + +class TestRowSourceDispatch: + def test_every_row_source_is_exercised_below(self): + """Guards the parametrization: a new RowSource no namespace here uses would go untested.""" + sources = {namespace_spec(namespace).source for namespace in EVERY_NAMESPACE} + + assert sources == set(RowSource) + + @pytest.mark.parametrize("namespace", EVERY_NAMESPACE) + def test_every_declared_source_resolves_to_a_row_datum(self, namespace): + spec = namespace_spec(namespace) + columns = {namespace: spec.columns(SAMPLE_DATASET_COLUMNS)} + + row = variant_to_csv_row(_Variant(), columns, namespaced=True) + + # Every planned column produced a cell. The parameterized namespaces are passed no per-row datum, + # so theirs are NA — the point here is that the dispatch reached them at all. + assert len(row) == len(columns[namespace]), namespace diff --git a/tests/lib/csv/test_variant.py b/tests/lib/csv/test_variant.py new file mode 100644 index 000000000..fd205f1cc --- /dev/null +++ b/tests/lib/csv/test_variant.py @@ -0,0 +1,1168 @@ +# ruff: noqa: E402 + +import csv +import io +from datetime import date +from unittest.mock import Mock, patch + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy import event + +from mavedb.lib.csv.namespaces import calibration_namespace_for_urn, is_valid_csv_namespace +from mavedb.lib.csv.score_set import available_score_set_csv_namespaces, get_score_set_variants_as_csv +from mavedb.lib.csv.variant import ( + BASE_VARIANT_CSV_NAMESPACES, + available_variant_csv_namespaces, + get_variant_csv, +) +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer +from mavedb.models.acmg_classification import ACMGClassification +from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.enums.acmg_criterion import ACMGCriterion +from mavedb.models.enums.user_role import UserRole +from mavedb.models.enums.functional_classification import FunctionalClassification as FunctionalClassificationOptions +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_set import ScoreSet +from mavedb.models.target_gene import TargetGene +from mavedb.models.variant import Variant +from tests.helpers.constants import ( + TEST_GNOMAD_DATA_VERSION, + TEST_GNOMAD_VARIANT, + TEST_MINIMAL_MAPPED_VARIANT, + TEST_MINIMAL_VARIANT, + TEST_SEQ_SCORESET, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _add_pathogenicity_calibration(db, score_set, variants_in_abnormal_range, urn, title, research_use_only=False): + """Attach a calibration with a normal (BS3) and abnormal (PS3) range to *score_set*. + + Only *variants_in_abnormal_range* are associated with the abnormal range, which is what + ``functional_classification_of_variant`` consults to classify a variant. + """ + calibration = ScoreCalibration( + score_set_id=score_set.id, + urn=urn, + title=title, + baseline_score=0.0, + research_use_only=research_use_only, + primary=True, + private=False, + calibration_metadata={}, + created_by_id=score_set.created_by_id, + modified_by_id=score_set.modified_by_id, + ) + db.add(calibration) + db.commit() + db.refresh(calibration) + + abnormal_acmg = db.query(ACMGClassification).filter(ACMGClassification.criterion == ACMGCriterion.PS3).first() + normal_acmg = db.query(ACMGClassification).filter(ACMGClassification.criterion == ACMGCriterion.BS3).first() + + db.add( + ScoreCalibrationFunctionalClassification( + calibration_id=calibration.id, + label="test abnormal functional range", + description="An abnormal functional range", + functional_classification=FunctionalClassificationOptions.abnormal, + range=[-5.0, -1.0], + inclusive_lower_bound=True, + inclusive_upper_bound=False, + acmg_classification_id=abnormal_acmg.id, + variants=list(variants_in_abnormal_range), + ) + ) + db.add( + ScoreCalibrationFunctionalClassification( + calibration_id=calibration.id, + label="test normal functional range", + description="A normal functional range", + functional_classification=FunctionalClassificationOptions.normal, + range=[1.0, 5.0], + inclusive_lower_bound=True, + inclusive_upper_bound=False, + acmg_classification_id=normal_acmg.id, + variants=[], + ) + ) + db.commit() + db.refresh(calibration) + + return calibration + + +def _add_rangeless_calibration(db, score_set, urn, title): + """Attach a calibration carrying only a baseline score, with no ranges to classify against. + + It can support neither a functional nor a pathogenicity annotation, so every column of its namespace + would be NA. + """ + calibration = ScoreCalibration( + score_set_id=score_set.id, + urn=urn, + title=title, + baseline_score=0.0, + research_use_only=False, + primary=True, + private=False, + calibration_metadata={}, + created_by_id=score_set.created_by_id, + modified_by_id=score_set.modified_by_id, + ) + db.add(calibration) + db.commit() + db.refresh(calibration) + + return calibration + + +def _add_second_score_set_with_equivalent_variant(db, first_score_set, clingen_allele_id): + """Create a second score set measuring the same ClinGen allele as *first_score_set*'s variant.""" + score_set_scaffold = TEST_SEQ_SCORESET.copy() + score_set_scaffold.pop("target_genes") + score_set = ScoreSet( + **score_set_scaffold, + urn="urn:mavedb:00000001-a-2", + experiment_id=first_score_set.experiment_id, + licence_id=first_score_set.licence_id, + created_by_id=first_score_set.created_by_id, + modified_by_id=first_score_set.modified_by_id, + ) + db.add(score_set) + db.commit() + db.refresh(score_set) + + variant = Variant(**TEST_MINIMAL_VARIANT, urn=f"{score_set.urn}#1", score_set_id=score_set.id) + db.add(variant) + db.commit() + db.refresh(variant) + + mapped_variant = MappedVariant( + **TEST_MINIMAL_MAPPED_VARIANT, + variant_id=variant.id, + clingen_allele_id=clingen_allele_id, + ) + db.add(mapped_variant) + db.commit() + db.refresh(mapped_variant) + + return score_set, variant, mapped_variant + + +def _add_clinvar_control(db, mapped_variant, significance, review_status, db_version): + mapped_variant.clinical_controls.append( + ClinicalControl( + db_identifier="183058", + gene_symbol="PTEN", + clinical_significance=significance, + clinical_review_status=review_status, + db_name="ClinVar", + db_version=db_version, + ) + ) + db.add(mapped_variant) + db.commit() + + +def _parse_csv(csv_text): + return list(csv.DictReader(io.StringIO(csv_text))) + + +# --------------------------------------------------------------------------- +# TestGetVariantCsv +# --------------------------------------------------------------------------- + +CALIBRATION_URN_1 = "urn:mavedb:calibration-11111111-1111-1111-1111-111111111111" +CALIBRATION_URN_2 = "urn:mavedb:calibration-22222222-2222-2222-2222-222222222222" +CALIBRATION_URN_OTHER_SCORE_SET = "urn:mavedb:calibration-33333333-3333-3333-3333-333333333333" + +CALIBRATION_NS_1 = calibration_namespace_for_urn(CALIBRATION_URN_1) +CALIBRATION_NS_2 = calibration_namespace_for_urn(CALIBRATION_URN_2) +CALIBRATION_NS_OTHER = calibration_namespace_for_urn(CALIBRATION_URN_OTHER_SCORE_SET) + + +class TestGetVariantCsv: + """Integration tests for the DB-bound clinical CSV composer.""" + + def test_single_variant_yields_one_row_of_base_columns(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert len(rows) == 1 + assert rows[0]["accession"] == variant.urn + assert rows[0]["relationship.match_type"] == "exact" + assert rows[0]["score_set.score_set_urn"] == variant.score_set.urn + assert rows[0]["scores.score"] == str(TEST_MINIMAL_VARIANT["data"]["score_data"]["score"]) + assert rows[0]["hgvs_nt"] == TEST_MINIMAL_VARIANT["hgvs_nt"] + + def test_unknown_urn_raises(self, session, setup_lib_db_with_mapped_variant): + with pytest.raises(ValueError, match="not found"): + get_variant_csv(session, "urn:mavedb:00000001-a-1#999") + + def test_no_calibration_yields_no_calibration_columns(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + + csv_text = get_variant_csv(session, variant.urn) + + assert "calibration." not in csv_text + + def test_calibration_namespace_is_included_by_default(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Test Clinical Calibration" + ) + + csv_text = get_variant_csv(session, variant.urn) + rows = _parse_csv(csv_text) + + assert len(rows) == 1 + assert rows[0][f"{CALIBRATION_NS_1}.title"] == "Test Clinical Calibration" + assert rows[0][f"{CALIBRATION_NS_1}.functional_classification"] == "abnormal" + assert rows[0][f"{CALIBRATION_NS_1}.acmg_criterion"] == "PS3" + assert rows[0][f"{CALIBRATION_NS_1}.acmg_evidence_strength"] == "STRONG" + assert rows[0][f"{CALIBRATION_NS_1}.acmg_evidence_outcome_code"] == "PS3" + assert rows[0][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "PATHOGENIC" + + def test_variant_outside_calibration_ranges_is_uncertain(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [], urn=CALIBRATION_URN_1, title="Test Clinical Calibration" + ) + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert rows[0][f"{CALIBRATION_NS_1}.functional_classification"] == "indeterminate" + assert rows[0][f"{CALIBRATION_NS_1}.acmg_evidence_strength"] == "NA" + assert rows[0][f"{CALIBRATION_NS_1}.acmg_evidence_outcome_code"] == "PS3_not_met" + assert rows[0][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "UNCERTAIN_SIGNIFICANCE" + + def test_multiple_calibrations_appear_side_by_side(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Contains The Variant" + ) + _add_pathogenicity_calibration( + session, variant.score_set, [], urn=CALIBRATION_URN_2, title="Does Not Contain The Variant" + ) + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert len(rows) == 1 + assert rows[0][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "PATHOGENIC" + assert rows[0][f"{CALIBRATION_NS_2}.pathogenicity_classification"] == "UNCERTAIN_SIGNIFICANCE" + + def test_research_use_only_calibration_is_offered_but_labelled(self, session, setup_lib_db_with_mapped_variant): + """The score set page already shows these, so the export offers them — flagged, not hidden.""" + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, + variant.score_set, + [variant], + urn=CALIBRATION_URN_1, + title="Provisional Calibration", + research_use_only=True, + ) + + entry = next( + entry + for entry in available_variant_csv_namespaces(session, variant.urn) + if entry.namespace == CALIBRATION_NS_1 + ) + + assert entry.label == "Research Use Only: Provisional Calibration" + assert entry.selected_by_default is False + + def test_research_use_only_calibration_is_not_in_the_default_download( + self, session, setup_lib_db_with_mapped_variant + ): + """A clinically-framed default must not silently carry unvalidated thresholds.""" + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, + variant.score_set, + [variant], + urn=CALIBRATION_URN_1, + title="Provisional Calibration", + research_use_only=True, + ) + + assert CALIBRATION_NS_1 not in get_variant_csv(session, variant.urn) + + def test_research_use_only_calibration_is_served_when_named(self, session, setup_lib_db_with_mapped_variant): + """Naming the namespace is the opt-in, and the exported row declares its own standing.""" + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, + variant.score_set, + [variant], + urn=CALIBRATION_URN_1, + title="Provisional Calibration", + research_use_only=True, + ) + + rows = _parse_csv(get_variant_csv(session, variant.urn, namespaces=["scores", CALIBRATION_NS_1])) + + assert rows[0][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "PATHOGENIC" + assert rows[0][f"{CALIBRATION_NS_1}.research_use_only"] == "True" + + def test_clinical_calibration_declares_it_is_not_research_use_only(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Clinical Calibration" + ) + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert rows[0][f"{CALIBRATION_NS_1}.research_use_only"] == "False" + + def test_explicit_namespaces_restrict_columns(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration(session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Requested") + _add_pathogenicity_calibration(session, variant.score_set, [variant], urn=CALIBRATION_URN_2, title="Omitted") + + csv_text = get_variant_csv(session, variant.urn, namespaces=["scores", CALIBRATION_NS_1]) + rows = _parse_csv(csv_text) + + assert rows[0][f"{CALIBRATION_NS_1}.title"] == "Requested" + assert CALIBRATION_NS_2 not in csv_text + # Namespaces the caller did not ask for contribute no columns. + assert "gnomad.gnomad_af" not in csv_text + assert "relationship.match_type" not in csv_text + + def test_equivalent_measurements_share_clingen_allele_id(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.clingen_allele_id = "CA123456" + session.add(mapped_variant) + session.commit() + + variant = mapped_variant.variant + _add_second_score_set_with_equivalent_variant(session, variant.score_set, "CA123456") + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert len(rows) == 2 + # The requested measurement comes first. + assert rows[0]["accession"] == variant.urn + assert rows[1]["score_set.score_set_urn"] == "urn:mavedb:00000001-a-2" + assert all(row["relationship.match_type"] == "exact" for row in rows) + assert all(row["clingen.clingen_allele_id"] == "CA123456" for row in rows) + + def test_each_measurement_is_interpreted_only_under_its_own_calibrations( + self, session, setup_lib_db_with_mapped_variant + ): + """A score from one assay carries no meaning under another assay's thresholds.""" + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.clingen_allele_id = "CA123456" + session.add(mapped_variant) + session.commit() + + variant = mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="First Score Set Calibration" + ) + other_score_set, other_variant, _ = _add_second_score_set_with_equivalent_variant( + session, variant.score_set, "CA123456" + ) + _add_pathogenicity_calibration( + session, + other_score_set, + [other_variant], + urn=CALIBRATION_URN_OTHER_SCORE_SET, + title="Second Score Set Calibration", + ) + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert len(rows) == 2 + # Each row is classified under its own score set's calibration... + assert rows[0][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "PATHOGENIC" + assert rows[1][f"{CALIBRATION_NS_OTHER}.pathogenicity_classification"] == "PATHOGENIC" + # ...and left empty under the other score set's. + assert rows[0][f"{CALIBRATION_NS_OTHER}.pathogenicity_classification"] == "NA" + assert rows[1][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "NA" + + def test_calibration_entries_report_their_score_set(self, session, setup_lib_db_with_mapped_variant): + """A calibration means nothing against another score set's scores, so say which one owns it.""" + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.clingen_allele_id = "CA123456" + session.add(mapped_variant) + session.commit() + + variant = mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="First Assay Calibration" + ) + other_score_set, other_variant, _ = _add_second_score_set_with_equivalent_variant( + session, variant.score_set, "CA123456" + ) + _add_pathogenicity_calibration( + session, + other_score_set, + [other_variant], + urn=CALIBRATION_URN_OTHER_SCORE_SET, + title="Second Assay Calibration", + ) + + by_namespace = {entry.namespace: entry for entry in available_variant_csv_namespaces(session, variant.urn)} + + first, second = by_namespace[CALIBRATION_NS_1], by_namespace[CALIBRATION_NS_OTHER] + assert first.score_set.urn == variant.score_set.urn + assert second.score_set.urn == other_score_set.urn + # The title is carried too, so a picker can name the score set rather than show its URN. + assert first.score_set.title == variant.score_set.title + # The two are distinguishable, which is the whole point. + assert first.score_set.urn != second.score_set.urn + + def test_non_calibration_entries_have_no_owning_score_set(self, session, setup_lib_db_with_mapped_variant): + """gnomAD and friends apply to any measurement, so there is no score set to attribute them to.""" + variant = setup_lib_db_with_mapped_variant.variant + + entries = available_variant_csv_namespaces(session, variant.urn) + + assert all(entry.score_set is None for entry in entries if not entry.namespace.startswith("calibration.")) + + def test_variant_without_clingen_allele_id_stands_alone(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + # A second measurement with a null allele ID must not be pulled in on a null match. + _add_second_score_set_with_equivalent_variant(session, variant.score_set, None) + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert len(rows) == 1 + assert rows[0]["accession"] == variant.urn + assert rows[0]["clingen.clingen_allele_id"] == "NA" + + def test_mapped_coordinates_and_external_annotations(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.hgvs_g = "NC_000010.11:g.87933147C>T" + mapped_variant.hgvs_c = "NM_000314.8:c.100A>G" + mapped_variant.hgvs_p = "NP_000305.3:p.Lys34Glu" + mapped_variant.vep_functional_consequence = "missense_variant" + mapped_variant.clingen_allele_id = "CA123456" + mapped_variant.gnomad_variants.append(GnomADVariant(**TEST_GNOMAD_VARIANT)) + session.add(mapped_variant) + session.commit() + + # Patched where it is used, not where it is defined: `fetch` binds the value with a `from` + # import, so patching `mavedb.lib.gnomad` would leave the query filtering on the real version. + with patch("mavedb.lib.csv.fetch.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) + + assert rows[0]["mavedb.post_mapped_hgvs_g"] == "NC_000010.11:g.87933147C>T" + assert rows[0]["mavedb.post_mapped_hgvs_c"] == "NM_000314.8:c.100A>G" + assert rows[0]["mavedb.post_mapped_hgvs_p"] == "NP_000305.3:p.Lys34Glu" + assert rows[0]["vep.vep_functional_consequence"] == "missense_variant" + assert rows[0]["gnomad.gnomad_af"] == str(TEST_GNOMAD_VARIANT["allele_frequency"]) + assert rows[0]["clingen.clingen_allele_id"] == "CA123456" + + def test_gnomad_variant_from_another_version_is_not_reported(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.gnomad_variants.append(GnomADVariant(**TEST_GNOMAD_VARIANT)) + session.add(mapped_variant) + session.commit() + + with patch("mavedb.lib.csv.fetch.GNOMAD_DATA_VERSION", "v9.9"): + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) + + # The variant still gets a row: the version predicate is in the join's ON clause, so a gnomAD + # record from another version leaves the frequency NA rather than dropping the variant. + assert len(rows) == 1 + assert rows[0]["gnomad.gnomad_af"] == "NA" + + def test_latest_clinvar_release_is_reported_and_labeled(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + _add_clinvar_control(session, mapped_variant, "Likely benign", "single submitter", "11_2024") + _add_clinvar_control(session, mapped_variant, "Pathogenic", "reviewed by expert panel", "02_2025") + + csv_text = get_variant_csv(session, mapped_variant.variant.urn) + rows = _parse_csv(csv_text) + + # The release is carried in the column name so the call stays citable. + assert "clinvar.2025_02.clinical_significance" in csv_text.splitlines()[0] + assert rows[0]["clinvar.2025_02.clinical_significance"] == "Pathogenic" + assert rows[0]["clinvar.2025_02.clinical_review_status"] == "reviewed by expert panel" + assert "clinvar.2024_11" not in csv_text + + def test_non_clinvar_control_is_not_reported(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.clinical_controls.append( + ClinicalControl( + db_identifier="ABC123", + gene_symbol="BRCA1", + clinical_significance="benign", + clinical_review_status="lots of convincing evidence", + db_name="GenDB", + db_version="2024", + ) + ) + session.add(mapped_variant) + session.commit() + + csv_text = get_variant_csv(session, mapped_variant.variant.urn) + + assert "clinvar" not in csv_text + assert "benign" not in csv_text + + def test_provenance_columns(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + session.add(TargetGene(score_set_id=variant.score_set.id, name="PTEN", category="protein_coding")) + session.commit() + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert rows[0]["score_set.score_set_urn"] == variant.score_set.urn + assert rows[0]["score_set.target_gene"] == "PTEN" + + def test_unmapped_variant_in_an_unmapped_score_set_omits_mapping_columns(self, session, setup_lib_db_with_variant): + """Nothing in this score set has been mapped, so those columns would say nothing at all.""" + variant = setup_lib_db_with_variant + + csv_text = get_variant_csv(session, variant.urn) + rows = _parse_csv(csv_text) + + assert len(rows) == 1 + assert rows[0]["accession"] == variant.urn + assert rows[0]["scores.score"] == str(TEST_MINIMAL_VARIANT["data"]["score_data"]["score"]) + for omitted in ( + "mavedb.post_mapped_hgvs_g", + "clingen.clingen_allele_id", + "gnomad.gnomad_af", + "vep.vep_functional_consequence", + ): + assert omitted not in csv_text + + def test_unmapped_variant_in_a_mapped_score_set_keeps_mapping_columns_as_na( + self, session, setup_lib_db_with_variant + ): + """The score set is mapped, so the columns exist for it; NA is the honest value for this variant.""" + variant = setup_lib_db_with_variant + mapped_sibling = Variant( + **{**TEST_MINIMAL_VARIANT, "urn": f"{variant.score_set.urn}#2"}, score_set_id=variant.score_set_id + ) + session.add(mapped_sibling) + session.commit() + session.add(MappedVariant(**TEST_MINIMAL_MAPPED_VARIANT, variant_id=mapped_sibling.id)) + session.commit() + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert len(rows) == 1 + assert rows[0]["accession"] == variant.urn + assert rows[0]["mavedb.post_mapped_hgvs_g"] == "NA" + assert rows[0]["clingen.clingen_allele_id"] == "NA" + assert rows[0]["gnomad.gnomad_af"] == "NA" + + def test_unmapped_variant_respects_requested_namespaces(self, session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + + csv_text = get_variant_csv(session, variant.urn, namespaces=["scores"]) + + assert "gnomad.gnomad_af" not in csv_text + assert "scores.score" in csv_text.splitlines()[0] + + def test_superseded_mapping_is_ignored(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.current = False + session.add(mapped_variant) + session.add( + MappedVariant( + **{**TEST_MINIMAL_MAPPED_VARIANT, "current": True}, + variant_id=mapped_variant.variant_id, + clingen_allele_id="CA999999", + ) + ) + session.commit() + + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) + + assert len(rows) == 1 + assert rows[0]["clingen.clingen_allele_id"] == "CA999999" + + def test_does_not_load_whole_score_range_variant_collections(self, session, setup_lib_db_with_mapped_variant): + """Range membership must come from the association table, not by loading every variant of a range. + + The ORM check in ``annotation.classification`` loads each range's entire variant collection — with + every variant's score data — once per range per row. On a large score set that dominates the + export's runtime, so this pins the cheap path rather than trusting it to stay. + """ + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration(session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="First") + _add_pathogenicity_calibration(session, variant.score_set, [], urn=CALIBRATION_URN_2, title="Second") + + statements: list[str] = [] + + def record(conn, cursor, statement, parameters, context, executemany): + statements.append(statement) + + bind = session.get_bind() + event.listen(bind, "before_cursor_execute", record) + try: + get_variant_csv(session, variant.urn) + finally: + event.remove(bind, "before_cursor_execute", record) + + collection_loads = [ + statement + for statement in statements + if "score_calibration_functional_classification_variants" in statement and "variants.data" in statement + ] + assert collection_loads == [], ( + f"{len(collection_loads)} range-collection load(s) during one export; " + "membership should come from the association table" + ) + + def test_namespace_discovery_does_not_scan_score_set_variants(self, session, setup_lib_db_with_mapped_variant): + """Discovery must key on score sets, not join through their variants. + + `ScoreSet.variants` multiplies the calibration join by every variant in the score set before + DISTINCT collapses it again, which made this route far slower than the score-set equivalent that + filters on score_set_id directly. + """ + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration(session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="First") + + statements: list[str] = [] + + def record(conn, cursor, statement, parameters, context, executemany): + statements.append(statement) + + bind = session.get_bind() + event.listen(bind, "before_cursor_execute", record) + try: + available_variant_csv_namespaces(session, variant.urn) + finally: + event.remove(bind, "before_cursor_execute", record) + + calibration_scans = [ + statement + for statement in statements + if "score_calibrations" in statement and " variants" in statement.replace("\n", " ") + ] + assert ( + calibration_scans == [] + ), "calibration discovery joined the variants table; it should filter on score_set_id" + + def test_base_namespaces_are_all_present_by_default(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + + header = _parse_csv(get_variant_csv(session, variant.urn))[0].keys() + + # One representative column per base namespace. + for column in ( + "scores.score", + "vep.vep_functional_consequence", + "gnomad.gnomad_af", + "clingen.clingen_allele_id", + "score_set.score_set_urn", + "relationship.match_type", + ): + assert column in header, f"{column} missing; BASE_VARIANT_CSV_NAMESPACES={BASE_VARIANT_CSV_NAMESPACES}" + + +# --------------------------------------------------------------------------- +# TestComputeAvailableCsvNamespaces +# --------------------------------------------------------------------------- + + +class TestComputeAvailableCsvNamespaces: + """What a namespace selector is offered for a score set.""" + + def test_mapped_score_set_offers_mapping_backed_namespaces(self, session, setup_lib_db_with_mapped_variant): + score_set = setup_lib_db_with_mapped_variant.variant.score_set + + namespaces = [entry.namespace for entry in available_score_set_csv_namespaces(session, score_set)] + + assert "score_set" in namespaces + assert {"vep", "gnomad", "clingen"} <= set(namespaces) + + def test_unmapped_score_set_omits_mapping_backed_namespaces(self, session, setup_lib_db_with_variant): + score_set = setup_lib_db_with_variant.score_set + + namespaces = [entry.namespace for entry in available_score_set_csv_namespaces(session, score_set)] + + assert "score_set" in namespaces + assert not {"vep", "gnomad", "clingen"} & set(namespaces) + + def test_relationship_is_never_offered(self, session, setup_lib_db_with_mapped_variant): + """match_type describes a row's relation to a requested record, which a score set has no notion of.""" + score_set = setup_lib_db_with_mapped_variant.variant.score_set + + assert "relationship" not in [ + entry.namespace for entry in available_score_set_csv_namespaces(session, score_set) + ] + + def test_score_and_count_namespaces_follow_dataset_columns(self, session, setup_lib_db_with_mapped_variant): + score_set = setup_lib_db_with_mapped_variant.variant.score_set + score_set.dataset_columns = {"score_columns": ["scores.score"], "count_columns": []} + session.add(score_set) + session.commit() + + namespaces = [entry.namespace for entry in available_score_set_csv_namespaces(session, score_set)] + + assert "scores" in namespaces + assert "counts" not in namespaces + + def test_calibration_namespaces_are_offered(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Clinical Calibration" + ) + + namespaces = [entry.namespace for entry in available_score_set_csv_namespaces(session, variant.score_set)] + + assert CALIBRATION_NS_1 in namespaces + + def test_research_use_only_calibration_is_offered_unchecked(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, + variant.score_set, + [variant], + urn=CALIBRATION_URN_1, + title="Provisional Calibration", + research_use_only=True, + ) + + entry = next( + entry + for entry in available_score_set_csv_namespaces(session, variant.score_set) + if entry.namespace == CALIBRATION_NS_1 + ) + + assert entry.label == "Research Use Only: Provisional Calibration" + assert entry.selected_by_default is False + # Reported in its own right, not left to be inferred from the label or from the unchecked box: + # this is the one reason for unchecking that decides whether the data may be published. + assert entry.research_use_only is True + + def test_clinical_calibration_is_offered_checked(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Clinical Calibration" + ) + + entry = next( + entry + for entry in available_score_set_csv_namespaces(session, variant.score_set) + if entry.namespace == CALIBRATION_NS_1 + ) + + assert entry.label == "Clinical Calibration" + assert entry.selected_by_default is True + assert entry.research_use_only is False + assert entry.score_set.urn == variant.score_set.urn + + def test_rangeless_calibration_is_offered_unchecked(self, session, setup_lib_db_with_mapped_variant): + """The score-set export covers the score set's own calibrations, so a rangeless one is still + requestable — but it would contribute nothing except NA, so it must not open checked and must not + reach the public dump, which takes only what discovery selects by default. + """ + variant = setup_lib_db_with_mapped_variant.variant + _add_rangeless_calibration(session, variant.score_set, urn=CALIBRATION_URN_1, title="Baseline Only") + + entry = next( + entry + for entry in available_score_set_csv_namespaces(session, variant.score_set) + if entry.namespace == CALIBRATION_NS_1 + ) + + assert entry.label == "Baseline Only" + assert entry.selected_by_default is False + + def test_variant_discovery_omits_a_rangeless_calibration_entirely(self, session, setup_lib_db_with_mapped_variant): + """A variant's calibrations are scoped to what interprets this allele; one that interprets + nothing is not a choice worth offering. + """ + variant = setup_lib_db_with_mapped_variant.variant + _add_rangeless_calibration(session, variant.score_set, urn=CALIBRATION_URN_1, title="Baseline Only") + + namespaces = [entry.namespace for entry in available_variant_csv_namespaces(session, variant.urn)] + + assert CALIBRATION_NS_1 not in namespaces + + def test_clinvar_namespaces_are_offered_per_release(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + _add_clinvar_control(session, mapped_variant, "Likely benign", "single submitter", "11_2024") + _add_clinvar_control(session, mapped_variant, "Pathogenic", "expert panel", "02_2025") + + namespaces = [ + entry.namespace for entry in available_score_set_csv_namespaces(session, mapped_variant.variant.score_set) + ] + + assert "clinvar.2024_11" in namespaces + assert "clinvar.2025_02" in namespaces + + def test_only_the_newest_clinvar_release_is_selected_by_default(self, session, setup_lib_db_with_mapped_variant): + """MaveDB carries around ten releases; a picker opening with all of them checked is unusable.""" + mapped_variant = setup_lib_db_with_mapped_variant + for db_version in ("11_2024", "02_2025", "06_2024"): + _add_clinvar_control(session, mapped_variant, "Pathogenic", "expert panel", db_version) + + by_namespace = { + entry.namespace: entry + for entry in available_score_set_csv_namespaces(session, mapped_variant.variant.score_set) + } + + assert by_namespace["clinvar.2025_02"].selected_by_default is True + assert by_namespace["clinvar.2024_11"].selected_by_default is False + assert by_namespace["clinvar.2024_06"].selected_by_default is False + # The older releases are still on offer — comparing a call across releases is a real thing to want. + assert len([ns for ns in by_namespace if ns.startswith("clinvar.")]) == 3 + + def test_every_offered_namespace_is_valid(self, session, setup_lib_db_with_mapped_variant): + """Discovery must only advertise namespaces the endpoints will actually accept.""" + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Clinical Calibration" + ) + _add_clinvar_control(session, setup_lib_db_with_mapped_variant, "Pathogenic", "expert panel", "02_2025") + + entries = available_score_set_csv_namespaces(session, variant.score_set) + + assert entries + assert all(is_valid_csv_namespace(entry.namespace) for entry in entries) + # Every entry must also be presentable, or a picker has nothing to render. + assert all(entry.label for entry in entries) + assert all(entry.group for entry in entries) + + def test_entries_are_labeled_for_a_picker(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Brnich et al. 2019" + ) + _add_clinvar_control(session, setup_lib_db_with_mapped_variant, "Pathogenic", "expert panel", "11_2024") + + by_namespace = { + entry.namespace: entry for entry in available_score_set_csv_namespaces(session, variant.score_set) + } + + # A calibration is named by its title, not its URN. + assert by_namespace[CALIBRATION_NS_1].label == "Brnich et al. 2019" + assert by_namespace[CALIBRATION_NS_1].group == "calibration" + # A ClinVar release is named by its date. + assert by_namespace["clinvar.2024_11"].label == "ClinVar significance (November 2024)" + assert by_namespace["clinvar.2024_11"].group == "annotation" + assert by_namespace["gnomad"].label == "gnomAD allele frequency" + assert by_namespace["score_set"].group == "provenance" + + +# --------------------------------------------------------------------------- +# TestAnchorMappingIsDeterministic +# --------------------------------------------------------------------------- + + +class TestAnchorMappingIsDeterministic: + """Everything in the CSV follows from which current mapping anchors the request.""" + + def test_repeat_downloads_agree_when_several_mappings_claim_to_be_current( + self, session, setup_lib_db_with_mapped_variant + ): + """Nothing in the schema stops two rows from being current, so the pick must not be arbitrary.""" + variant = setup_lib_db_with_mapped_variant.variant + + newer = MappedVariant( + **{**TEST_MINIMAL_MAPPED_VARIANT, "mapped_date": date(2030, 1, 1), "clingen_allele_id": "CA_NEWER"}, + variant_id=variant.id, + ) + session.add(newer) + session.commit() + + first = _parse_csv(get_variant_csv(session, variant.urn, ["clingen"])) + second = _parse_csv(get_variant_csv(session, variant.urn, ["clingen"])) + + assert first == second + # The requested variant anchors the export and comes first, so this pins which mapping was + # picked rather than merely whether the newer one appears anywhere in the output. + assert first[0]["clingen.clingen_allele_id"] == "CA_NEWER" + + def test_a_variant_with_two_current_mappings_is_reported_once(self, session, setup_lib_db_with_mapped_variant): + """Two current mappings on one variant are the same measurement twice, not two equivalents. + + Emitting both would also break the row ordering downstream, which restores the caller's order from + the variant ids alone. + """ + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.clingen_allele_id = "CA123456" + session.add(mapped_variant) + session.commit() + + variant = mapped_variant.variant + session.add( + MappedVariant( + **{**TEST_MINIMAL_MAPPED_VARIANT, "mapped_date": date(2020, 1, 1), "clingen_allele_id": "CA123456"}, + variant_id=variant.id, + ) + ) + session.commit() + + # A genuine equivalent in another score set, so the widening is doing something to dedupe within. + _add_second_score_set_with_equivalent_variant(session, variant.score_set, "CA123456") + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert [row["accession"] for row in rows] == [variant.urn, "urn:mavedb:00000001-a-2#1"] + + def test_an_equivalent_variants_extra_current_mapping_is_reported_once( + self, session, setup_lib_db_with_mapped_variant + ): + """The same rule applies to the widened rows, not just to the anchor.""" + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.clingen_allele_id = "CA123456" + session.add(mapped_variant) + session.commit() + + variant = mapped_variant.variant + _, other_variant, _ = _add_second_score_set_with_equivalent_variant(session, variant.score_set, "CA123456") + session.add( + MappedVariant( + **{**TEST_MINIMAL_MAPPED_VARIANT, "mapped_date": date(2020, 1, 1), "clingen_allele_id": "CA123456"}, + variant_id=other_variant.id, + ) + ) + session.commit() + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert [row["accession"] for row in rows] == [variant.urn, other_variant.urn] + + +# --------------------------------------------------------------------------- +# TestScoreSetCsvCalibrationColumns +# --------------------------------------------------------------------------- + + +class TestScoreSetCsvCalibrationColumns: + """The score-set CSV must fill the calibration columns its discovery advertises.""" + + def test_calibration_columns_are_populated(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Clinical Calibration" + ) + + rows = _parse_csv( + get_score_set_variants_as_csv(session, variant.score_set, ["scores", CALIBRATION_NS_1], namespaced=True) + ) + + row = next(r for r in rows if r["accession"] == variant.urn) + assert row[f"{CALIBRATION_NS_1}.title"] == "Clinical Calibration" + assert row[f"{CALIBRATION_NS_1}.acmg_criterion"] == "PS3" + assert row[f"{CALIBRATION_NS_1}.acmg_evidence_outcome_code"] == "PS3" + assert row[f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "PATHOGENIC" + assert row[f"{CALIBRATION_NS_1}.research_use_only"] == "False" + + def test_variant_outside_the_range_is_uncertain_not_blank(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration(session, variant.score_set, [], urn=CALIBRATION_URN_1, title="Clinical") + + rows = _parse_csv( + get_score_set_variants_as_csv(session, variant.score_set, ["scores", CALIBRATION_NS_1], namespaced=True) + ) + + row = next(r for r in rows if r["accession"] == variant.urn) + assert row[f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "UNCERTAIN_SIGNIFICANCE" + assert row[f"{CALIBRATION_NS_1}.acmg_evidence_outcome_code"] == "PS3_not_met" + + def test_rangeless_calibration_reports_its_identity_with_no_interpretation( + self, session, setup_lib_db_with_mapped_variant + ): + """It cannot classify anything, but which calibration was consulted is still on the record. + + The public dump carries these namespaces, so a wholly-NA block would be the archive claiming to + know less than the database does. + """ + variant = setup_lib_db_with_mapped_variant.variant + _add_rangeless_calibration(session, variant.score_set, urn=CALIBRATION_URN_1, title="Baseline Only") + + rows = _parse_csv( + get_score_set_variants_as_csv(session, variant.score_set, ["scores", CALIBRATION_NS_1], namespaced=True) + ) + + row = next(r for r in rows if r["accession"] == variant.urn) + assert row[f"{CALIBRATION_NS_1}.title"] == "Baseline Only" + assert row[f"{CALIBRATION_NS_1}.research_use_only"] == "False" + assert row[f"{CALIBRATION_NS_1}.functional_classification"] == "NA" + assert row[f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "NA" + + def test_everything_discovery_advertises_is_actually_populated(self, session, setup_lib_db_with_mapped_variant): + """Discovery and the export must agree, or a dump ships documented but empty columns.""" + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Clinical Calibration" + ) + + advertised = [ + entry.namespace + for entry in available_score_set_csv_namespaces(session, variant.score_set) + if entry.namespace.startswith("calibration.") + ] + assert advertised, "no calibration namespace advertised; the rest of this test proves nothing" + + rows = _parse_csv( + get_score_set_variants_as_csv(session, variant.score_set, ["scores"] + advertised, namespaced=True) + ) + row = next(r for r in rows if r["accession"] == variant.urn) + + for namespace in advertised: + assert row[f"{namespace}.title"] != "NA", f"{namespace} advertised but its columns are empty" + + def test_counts_are_always_taken_in_full(self, session, setup_lib_db_with_mapped_variant): + """Counts have no required column, so nothing narrows them the way ``scores`` is narrowed.""" + score_set = setup_lib_db_with_mapped_variant.variant.score_set + score_set.dataset_columns = {"score_columns": ["score"], "count_columns": ["c_0"]} + session.add(score_set) + session.commit() + + csv_text = get_score_set_variants_as_csv(session, score_set, ["counts"], namespaced=True) + + assert "counts.c_0" in csv_text.splitlines()[0] + + +class TestPrivateCalibrationsAreNotDisclosed: + """A calibration's READ permission is stricter than its score set's. + + Private ones are readable only by their owner, by contributors when investigator-provided, or by an + admin, so reading the measurement does not entitle a caller to the interpretation. + """ + + @pytest.fixture + def private_calibration(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + calibration = _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Unpublished Calibration" + ) + calibration.private = True + session.add(calibration) + session.commit() + return calibration + + def test_score_set_discovery_omits_it_by_default(self, session, private_calibration): + namespaces = [ + entry.namespace for entry in available_score_set_csv_namespaces(session, private_calibration.score_set) + ] + + assert CALIBRATION_NS_1 not in namespaces + + def test_variant_discovery_omits_it_by_default( + self, session, setup_lib_db_with_mapped_variant, private_calibration + ): + variant = setup_lib_db_with_mapped_variant.variant + + namespaces = [entry.namespace for entry in available_variant_csv_namespaces(session, variant.urn)] + + assert CALIBRATION_NS_1 not in namespaces + + def test_naming_the_urn_directly_yields_no_interpretation( + self, session, setup_lib_db_with_mapped_variant, private_calibration + ): + """Discovery is not the gate: a caller who knows the URN must still be refused the data.""" + variant = setup_lib_db_with_mapped_variant.variant + + rows = _parse_csv(get_variant_csv(session, variant.urn, ["scores", CALIBRATION_NS_1])) + + assert rows[0][f"{CALIBRATION_NS_1}.title"] == "NA" + assert rows[0][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "NA" + + def test_score_set_csv_withholds_it_too(self, session, private_calibration): + rows = _parse_csv( + get_score_set_variants_as_csv( + session, private_calibration.score_set, ["scores", CALIBRATION_NS_1], namespaced=True + ) + ) + + assert all(row[f"{CALIBRATION_NS_1}.title"] == "NA" for row in rows) + + def test_a_permitted_caller_still_receives_it(self, session, setup_lib_db_with_mapped_variant, private_calibration): + """Viewer-scoped emission: the viewer widens access, it is not a blanket ban on private calibrations. + + Uses a real entitled viewer rather than an always-true stand-in, so this exercises the same + ``ScoreCalibrationViewer`` rule the routers use. + """ + variant = setup_lib_db_with_mapped_variant.variant + admin = Principal(Mock(user=Mock(id=1, username="admin"), active_roles=[UserRole.admin])) + + rows = _parse_csv( + get_variant_csv( + session, + variant.urn, + ["scores", CALIBRATION_NS_1], + viewer=admin.viewer_for(ScoreCalibrationViewer), + ) + ) + + assert rows[0][f"{CALIBRATION_NS_1}.title"] == "Unpublished Calibration" + + def test_the_public_export_never_carries_it(self, session, private_calibration): + """The dump has no caller, so the default must be the public subset.""" + from mavedb.scripts.export_public_data import annotation_export_namespaces + + assert CALIBRATION_NS_1 not in annotation_export_namespaces(session, private_calibration.score_set) + + +class TestScoreColumnNamespaces: + """`scores` is the required column; `scores_custom` is the rest, emitted under the same prefix.""" + + @pytest.fixture + def score_set_with_custom_columns(self, session, setup_lib_db_with_mapped_variant): + score_set = setup_lib_db_with_mapped_variant.variant.score_set + score_set.dataset_columns = {"score_columns": ["score", "se"], "count_columns": []} + session.add(score_set) + session.commit() + return score_set + + def test_scores_alone_emits_only_the_required_column(self, session, score_set_with_custom_columns): + header = _parse_csv( + get_score_set_variants_as_csv(session, score_set_with_custom_columns, ["scores"], namespaced=True) + )[0] + + assert "scores.score" in header + assert "scores.se" not in header + + def test_custom_columns_are_emitted_under_the_scores_prefix(self, session, score_set_with_custom_columns): + """The published header must not change: `scores_custom` is a request token, not a column prefix.""" + header = _parse_csv( + get_score_set_variants_as_csv(session, score_set_with_custom_columns, ["scores_custom"], namespaced=True) + )[0] + + assert "scores.se" in header + assert not any(column.startswith("scores_custom.") for column in header) + + def test_both_namespaces_reproduce_the_whole_score_group_in_order(self, session, score_set_with_custom_columns): + header = list( + _parse_csv( + get_score_set_variants_as_csv( + session, score_set_with_custom_columns, ["scores", "scores_custom"], namespaced=True + ) + )[0] + ) + + assert [column for column in header if column.startswith("scores.")] == ["scores.score", "scores.se"] + + def test_discovery_offers_custom_columns_only_when_there_are_any( + self, session, score_set_with_custom_columns, setup_lib_db_with_mapped_variant + ): + offered = [ + entry.namespace for entry in available_score_set_csv_namespaces(session, score_set_with_custom_columns) + ] + assert "scores" in offered and "scores_custom" in offered + + score_set_with_custom_columns.dataset_columns = {"score_columns": ["score"], "count_columns": []} + session.add(score_set_with_custom_columns) + session.commit() + + offered = [ + entry.namespace for entry in available_score_set_csv_namespaces(session, score_set_with_custom_columns) + ] + assert "scores" in offered and "scores_custom" not in offered diff --git a/tests/lib/mave/__init__.py b/tests/lib/mave/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lib/test_acmg.py b/tests/lib/test_acmg.py index cc5dfac0c..bf82e3629 100644 --- a/tests/lib/test_acmg.py +++ b/tests/lib/test_acmg.py @@ -6,6 +6,7 @@ pytest.importorskip("psycopg2") from mavedb.lib.acmg import ( + acmg_evidence_outcome_code, ACMGCriterion, StrengthOfEvidenceProvided, find_or_create_acmg_classification, @@ -241,3 +242,36 @@ def test_find_or_create_acmg_classification_does_not_commit(session): ).scalar_one_or_none() assert existing is None + + +######################################################################################################################## +# Tests for acmg_evidence_outcome_code +######################################################################################################################## + + +@pytest.mark.parametrize( + "criterion, evidence_strength, expected", + [ + # STRONG is a criterion's baseline, so it is written bare. + ("PS3", "STRONG", "PS3"), + ("BS3", "STRONG", "BS3"), + # Anything else is suffixed. + ("PS3", "VERY_STRONG", "PS3_very_strong"), + ("PS3", "MODERATE", "PS3_moderate"), + ("PS3", "SUPPORTING", "PS3_supporting"), + ("BS3", "SUPPORTING", "BS3_supporting"), + # MaveDB's intermediate strength has no VA-Spec equivalent, but the code format is the same. + ("PS3", "MODERATE_PLUS", "PS3_moderate_plus"), + # No strength means the criterion was evaluated and not met. + ("PS3", None, "PS3_not_met"), + ("BS3", None, "BS3_not_met"), + ], +) +def test_acmg_evidence_outcome_code(criterion, evidence_strength, expected): + assert acmg_evidence_outcome_code(criterion, evidence_strength) == expected + + +def test_acmg_evidence_outcome_code_is_case_insensitive_about_strength(): + """Callers pass a name from whichever enumeration they hold; casing should not change the result.""" + assert acmg_evidence_outcome_code("PS3", "strong") == "PS3" + assert acmg_evidence_outcome_code("PS3", "Moderate") == "PS3_moderate" diff --git a/tests/lib/test_score_set.py b/tests/lib/test_score_set.py index 3ca40d4a6..53d1e874b 100644 --- a/tests/lib/test_score_set.py +++ b/tests/lib/test_score_set.py @@ -22,7 +22,6 @@ create_variants_data, csv_data_to_df, fetch_score_set_search_filter_options, - variant_to_csv_row, ) from mavedb.lib.types.authentication import UserData from mavedb.lib.validation.constants.general import ( @@ -556,154 +555,3 @@ def test_fetch_score_set_search_filter_options_with_no_permitted_score_sets(setu "publication_db_names": [], "publication_journals": [], } - - -class MockVariant: - """Lightweight mock for Variant used in variant_to_csv_row tests.""" - - def __init__(self, urn="urn:mavedb:00000001-a-1#1", hgvs_nt=None, hgvs_splice=None, hgvs_pro=None, data=None): - self.urn = urn - self.hgvs_nt = hgvs_nt - self.hgvs_splice = hgvs_splice - self.hgvs_pro = hgvs_pro - self.data = data - - -class TestVariantToCsvRowNullHandling: - """Tests that variant_to_csv_row represents missing data as na_rep, not 'None'.""" - - def test_score_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"score_data": {"score": None}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_score_data_with_missing_key_uses_na_rep(self): - variant = MockVariant(data={"score_data": {}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_score_data_with_no_score_data_key_uses_na_rep(self): - variant = MockVariant(data={}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_score_data_with_no_data_uses_na_rep(self): - variant = MockVariant(data=None) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_count_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"count_data": {"count1": None}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_count_data_with_missing_key_uses_na_rep(self): - variant = MockVariant(data={"count_data": {}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_count_data_with_no_count_data_key_uses_na_rep(self): - variant = MockVariant(data={}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_count_data_with_no_data_uses_na_rep(self): - variant = MockVariant(data=None) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_score_data_with_valid_value_preserved(self): - variant = MockVariant(data={"score_data": {"score": 1.5}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "1.5" - - def test_count_data_with_valid_value_preserved(self): - variant = MockVariant(data={"count_data": {"count1": 42}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "42" - - def test_score_data_with_custom_na_rep(self): - variant = MockVariant(data={"score_data": {"score": None}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns, na_rep="N/A") - - assert row["score"] == "N/A" - - def test_namespaced_score_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"score_data": {"score": None}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns, namespaced=True) - - assert row["scores.score"] == "NA" - - def test_namespaced_count_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"count_data": {"count1": None}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns, namespaced=True) - - assert row["counts.count1"] == "NA" - - def test_core_columns_with_none_hgvs_uses_na_rep(self): - variant = MockVariant(hgvs_nt=None, hgvs_pro=None, hgvs_splice=None, urn="urn:mavedb:00000001-a-1#1") - columns = {"core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"]} - - row = variant_to_csv_row(variant, columns) - - assert row["hgvs_nt"] == "NA" - assert row["hgvs_pro"] == "NA" - assert row["hgvs_splice"] == "NA" - assert row["accession"] == "urn:mavedb:00000001-a-1#1" - - def test_mixed_columns_with_missing_data(self): - variant = MockVariant( - hgvs_nt="g.1A>G", - hgvs_pro="p.Met1Val", - data={"score_data": {"score": None, "se": 0.1}, "count_data": {"count1": None, "count2": 5}}, - ) - columns = { - "core": ["hgvs_nt", "hgvs_pro"], - "scores": ["score", "se"], - "counts": ["count1", "count2"], - } - - row = variant_to_csv_row(variant, columns) - - assert row["hgvs_nt"] == "g.1A>G" - assert row["hgvs_pro"] == "p.Met1Val" - assert row["score"] == "NA" - assert row["se"] == "0.1" - assert row["count1"] == "NA" - assert row["count2"] == "5" diff --git a/tests/lib/test_urns.py b/tests/lib/test_urns.py new file mode 100644 index 000000000..380bf0fc9 --- /dev/null +++ b/tests/lib/test_urns.py @@ -0,0 +1,99 @@ +import pytest + +from mavedb.lib.urns import score_set_urn_sort_key, variant_urn_sort_key + +pytestmark = pytest.mark.unit + + +# --------------------------------------------------------------------------- +# Each of these pairs is one a lexical sort gets wrong. They are the reason the keys exist, so they are +# asserted against plain string ordering too — a test that only checked the key would keep passing if +# someone decided the URNs sort fine on their own. +# --------------------------------------------------------------------------- + + +class TestScoreSetUrnSortKey: + def test_unpadded_score_set_number_orders_numerically(self): + urns = ["urn:mavedb:00000001-a-10", "urn:mavedb:00000001-a-2"] + + assert sorted(urns) == ["urn:mavedb:00000001-a-10", "urn:mavedb:00000001-a-2"] + assert sorted(urns, key=score_set_urn_sort_key) == [ + "urn:mavedb:00000001-a-2", + "urn:mavedb:00000001-a-10", + ] + + def test_experiment_suffix_orders_by_length_then_alphabetically(self): + """MaveDB assigns experiment suffixes a..z then aa..az, so `z` precedes `aa`.""" + urns = ["urn:mavedb:00000001-aa-1", "urn:mavedb:00000001-b-1", "urn:mavedb:00000001-z-1"] + + assert sorted(urns)[0] == "urn:mavedb:00000001-aa-1" + assert sorted(urns, key=score_set_urn_sort_key) == [ + "urn:mavedb:00000001-b-1", + "urn:mavedb:00000001-z-1", + "urn:mavedb:00000001-aa-1", + ] + + def test_experiment_sets_order_before_their_experiments(self): + urns = ["urn:mavedb:00000002-a-1", "urn:mavedb:00000001-z-9"] + + assert sorted(urns, key=score_set_urn_sort_key) == [ + "urn:mavedb:00000001-z-9", + "urn:mavedb:00000002-a-1", + ] + + def test_zero_experiment_suffix_is_accepted(self): + """The experiment URN grammar allows a literal `0` alongside the letter suffixes.""" + assert score_set_urn_sort_key("urn:mavedb:00000001-0-1")[0] == 0 + + @pytest.mark.parametrize( + "urn", + [ + # An unpublished score set. A SQL cast on the suffix would be handed "467a" and error. + "tmp:8f14e45f-ceea-467a-9c4f-0b1d2e3f4a5b", + "not a urn at all", + "", + None, + ], + ) + def test_undecomposable_urns_sort_after_published_ones_without_raising(self, urn): + published = "urn:mavedb:00000001-a-1" + + assert sorted([urn, published], key=score_set_urn_sort_key) == [published, urn] + + def test_undecomposable_urns_still_order_deterministically_among_themselves(self): + urns = ["tmp:b", "tmp:a", "tmp:c"] + + assert sorted(urns, key=score_set_urn_sort_key) == ["tmp:a", "tmp:b", "tmp:c"] + + +class TestVariantUrnSortKey: + def test_unpadded_variant_number_orders_numerically(self): + urns = [f"urn:mavedb:00000001-a-1#{n}" for n in (2, 10, 1)] + + assert sorted(urns)[0] == "urn:mavedb:00000001-a-1#1" + assert sorted(urns)[1] == "urn:mavedb:00000001-a-1#10" + assert sorted(urns, key=variant_urn_sort_key) == [ + "urn:mavedb:00000001-a-1#1", + "urn:mavedb:00000001-a-1#2", + "urn:mavedb:00000001-a-1#10", + ] + + def test_variants_group_by_score_set_before_their_number(self): + urns = ["urn:mavedb:00000001-a-2#1", "urn:mavedb:00000001-a-1#3"] + + assert sorted(urns, key=variant_urn_sort_key) == [ + "urn:mavedb:00000001-a-1#3", + "urn:mavedb:00000001-a-2#1", + ] + + def test_unpublished_variant_urn_still_orders_by_its_number(self): + """A variant of an unpublished score set is `tmp:#N`, so the suffix is still parseable.""" + urns = ["tmp:abc#10", "tmp:abc#2"] + + assert sorted(urns, key=variant_urn_sort_key) == ["tmp:abc#2", "tmp:abc#10"] + + @pytest.mark.parametrize("urn", ["urn:mavedb:00000001-a-1", "", None]) + def test_urns_without_a_variant_suffix_sort_last_without_raising(self, urn): + numbered = "urn:mavedb:00000001-a-1#1" + + assert sorted([urn, numbered], key=variant_urn_sort_key) == [numbered, urn] diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index b7ecb514d..7fb80c12c 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -3428,7 +3428,7 @@ def test_download_variants_data_file( worker_queue.assert_called_once() download_scores_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?drop_na_columns=true&include_post_mapped_hgvs=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?drop_unused_hgvs_columns=true&namespaces=scores&namespaces=mavedb" ) assert download_scores_csv_response.status_code == 200 download_scores_csv = download_scores_csv_response.text @@ -3477,7 +3477,7 @@ def test_download_scores_file(session, data_provider, client, setup_router_db, d worker_queue.assert_called_once() download_scores_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/scores?drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/scores?drop_unused_hgvs_columns=true" ) assert download_scores_csv_response.status_code == 200 download_scores_csv = download_scores_csv_response.text @@ -3499,7 +3499,7 @@ def test_download_counts_file(session, data_provider, client, setup_router_db, d worker_queue.assert_called_once() download_counts_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/counts?drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/counts?drop_unused_hgvs_columns=true" ) assert download_counts_csv_response.status_code == 200 download_counts_csv = download_counts_csv_response.text @@ -3510,6 +3510,144 @@ def test_download_counts_file(session, data_provider, client, setup_router_db, d assert "hgvs_splice" not in columns +# Deprecated query-parameter aliases. Galaxy and other external tooling call these endpoints, so the old +# names keep working for a release rather than being silently ignored. +def test_deprecated_drop_na_columns_still_drops_unused_hgvs_columns( + session, data_provider, client, setup_router_db, data_files +): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + for path in ("variants/data?namespaces=scores&", "scores?", "counts?"): + response = client.get(f"/api/v1/score-sets/{published_score_set['urn']}/{path}drop_na_columns=true") + + assert response.status_code == 200, path + columns = response.text.split("\n")[0].split(",") + assert "hgvs_splice" not in columns, path + + +def test_deprecated_include_post_mapped_hgvs_adds_the_mavedb_namespace( + session, data_provider, client, setup_router_db, data_files +): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + create_mapped_variants_for_score_set(session, score_set["urn"], TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION) + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + response = client.get( + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&include_post_mapped_hgvs=true" + ) + + assert response.status_code == 200 + columns = response.text.split("\n")[0].split(",") + # Additive, as the flag always was: the requested namespace survives alongside it. + assert "scores.score" in columns + assert "mavedb.post_mapped_hgvs_g" in columns + + +def test_deprecated_include_custom_columns_adds_the_scores_custom_namespace( + session, data_provider, client, setup_router_db, data_files +): + """The flag now appends a namespace, and its columns keep the `scores.` prefix they always had.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + with_flag = client.get( + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&include_custom_columns=true" + ) + with_namespace = client.get( + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&namespaces=scores_custom" + ) + + assert with_flag.status_code == 200 + assert with_namespace.status_code == 200 + assert with_flag.text.split("\n")[0] == with_namespace.text.split("\n")[0] + assert with_flag.headers["Deprecation"] == "true" + assert "include_custom_columns is deprecated" in with_flag.headers["Warning"] + # No column is emitted under a `scores_custom.` prefix; the namespace is a request token only. + assert "scores_custom." not in with_flag.text + + +def test_current_parameter_name_wins_over_its_deprecated_spelling( + session, data_provider, client, setup_router_db, data_files +): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + response = client.get( + f"/api/v1/score-sets/{published_score_set['urn']}/scores?drop_unused_hgvs_columns=false&drop_na_columns=true" + ) + + assert response.status_code == 200 + assert "hgvs_splice" in response.text.split("\n")[0].split(",") + + +def test_deprecated_request_answers_with_deprecation_headers( + session, data_provider, client, setup_router_db, data_files +): + """The consumers here are scripts, not people reading our logs, so the response has to say so.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + response = client.get( + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data" + "?namespaces=scores&drop_na_columns=true&include_post_mapped_hgvs=true" + ) + + assert response.status_code == 200 + assert response.headers["deprecation"] == "true" + warning = response.headers["warning"] + assert "drop_na_columns is deprecated, use drop_unused_hgvs_columns" in warning + assert "include_post_mapped_hgvs is deprecated, use namespaces=mavedb" in warning + + +def test_current_request_carries_no_deprecation_headers(session, data_provider, client, setup_router_db, data_files): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + response = client.get( + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&drop_unused_hgvs_columns=true" + ) + + assert response.status_code == 200 + assert "deprecation" not in response.headers + assert "warning" not in response.headers + + +def test_deprecated_parameters_are_marked_deprecated_in_the_openapi_schema(client): + """Anyone reading the docs or generating a client should see the deprecation without sending a request.""" + schema = client.app.openapi() + + def parameter(path: str, name: str): + return next(p for p in schema["paths"][path]["get"]["parameters"] if p["name"] == name) + + for path, name in ( + ("/api/v1/score-sets/{urn}/variants/data", "drop_na_columns"), + ("/api/v1/score-sets/{urn}/variants/data", "include_post_mapped_hgvs"), + ("/api/v1/score-sets/{urn}/scores", "drop_na_columns"), + ("/api/v1/score-sets/{urn}/counts", "drop_na_columns"), + ): + assert parameter(path, name)["deprecated"] is True, f"{name} on {path}" + assert "deprecated" in parameter(path, name)["description"].lower(), f"{name} on {path}" + + # Namespace variant CSV export tests. def test_download_scores_file_in_variant_data_path(session, data_provider, client, setup_router_db, data_files): experiment = create_experiment(client) @@ -3522,7 +3660,7 @@ def test_download_scores_file_in_variant_data_path(session, data_provider, clien worker_queue.assert_called_once() download_scores_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&drop_unused_hgvs_columns=true" ) assert download_scores_csv_response.status_code == 200 download_scores_csv = download_scores_csv_response.text @@ -3545,7 +3683,7 @@ def test_download_counts_file_in_variant_data_path(session, data_provider, clien worker_queue.assert_called_once() download_counts_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=counts&include_custom_columns=true&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=counts&include_custom_columns=true&drop_unused_hgvs_columns=true" ) assert download_counts_csv_response.status_code == 200 download_counts_csv = download_counts_csv_response.text @@ -3569,7 +3707,7 @@ def test_download_scores_and_counts_file(session, data_provider, client, setup_r worker_queue.assert_called_once() download_scores_and_counts_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=counts&namespaces=scores&include_custom_columns=true&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=counts&namespaces=scores&include_custom_columns=true&drop_unused_hgvs_columns=true" ) assert download_scores_and_counts_csv_response.status_code == 200 download_scores_and_counts_csv = download_scores_and_counts_csv_response.text @@ -3604,7 +3742,7 @@ def test_download_scores_counts_and_post_mapped_variants_file( worker_queue.assert_called_once() download_multiple_data_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&namespaces=counts&include_custom_columns=true&include_post_mapped_hgvs=true&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&namespaces=counts&namespaces=mavedb&include_custom_columns=true&drop_unused_hgvs_columns=true" ) assert download_multiple_data_csv_response.status_code == 200 download_multiple_data_csv = download_multiple_data_csv_response.text @@ -3643,7 +3781,7 @@ def test_download_vep_file_in_variant_data_path(session, data_provider, client, worker_queue.assert_called_once() response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=vep&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=vep&drop_unused_hgvs_columns=true" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -3672,7 +3810,7 @@ def test_download_clingen_file_in_variant_data_path(session, data_provider, clie worker_queue.assert_called_once() response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=clingen&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=clingen&drop_unused_hgvs_columns=true" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -3694,13 +3832,39 @@ def test_download_gnomad_file_in_variant_data_path(session, data_provider, clien worker_queue.assert_called_once() response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=gnomad&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=gnomad&drop_unused_hgvs_columns=true" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) assert "gnomad.gnomad_af" in reader.fieldnames +def test_download_gnomad_file_keeps_variants_linked_to_other_gnomad_versions( + session, data_provider, client, setup_router_db, data_files +): + """A variant linked only to a gnomAD record of another version must still appear, with an NA frequency. + + The version filter belongs in the join's ON clause; in a WHERE it silently drops the variant row. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + # The seeded gnomAD variant's version deliberately differs from the configured export version. + link_gnomad_variants_to_mapped_variants(session, score_set) + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: + published_score_set = publish_score_set(client, score_set["urn"]) + worker_queue.assert_called_once() + + response = client.get(f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=gnomad") + assert response.status_code == 200 + + rows = list(csv.DictReader(StringIO(response.text))) + assert len(rows) == 3, "every variant must be present regardless of linked gnomAD versions" + assert all(row["gnomad.gnomad_af"] == "NA" for row in rows) + + def test_download_clingen_and_vep_file_in_variant_data_path( session, data_provider, client, setup_router_db, data_files ): @@ -3722,7 +3886,7 @@ def test_download_clingen_and_vep_file_in_variant_data_path( worker_queue.assert_called_once() response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=clingen&namespaces=vep&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=clingen&namespaces=vep&drop_unused_hgvs_columns=true" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -3754,7 +3918,7 @@ def test_download_clingen_and_scores_file_in_variant_data_path( worker_queue.assert_called_once() response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&namespaces=clingen&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&namespaces=clingen&drop_unused_hgvs_columns=true" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -3790,7 +3954,7 @@ def test_download_clinvar_namespace_in_variant_data_path(session, data_provider, response = client.get( f"/api/v1/score-sets/{published_score_set['urn']}/variants/data" - f"?namespaces={clinvar_namespace}&drop_na_columns=false" + f"?namespaces={clinvar_namespace}&drop_unused_hgvs_columns=false" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -3824,7 +3988,7 @@ def test_download_clinvar_namespace_with_no_matching_version( response = client.get( f"/api/v1/score-sets/{published_score_set['urn']}/variants/data" - f"?namespaces={clinvar_namespace}&drop_na_columns=false" + f"?namespaces={clinvar_namespace}&drop_unused_hgvs_columns=false" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -3854,7 +4018,7 @@ def test_download_multiple_clinvar_namespaces_in_variant_data_path( response = client.get( f"/api/v1/score-sets/{published_score_set['urn']}/variants/data" - f"?namespaces={matching_ns}&namespaces={non_matching_ns}&drop_na_columns=false" + f"?namespaces={matching_ns}&namespaces={non_matching_ns}&drop_unused_hgvs_columns=false" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -4066,7 +4230,7 @@ def test_cannot_get_annotated_variants_for_score_set_with_no_mapped_variants( publish_score_set = publish_score_set_response.json() download_scores_csv_response = client.get( - f"/api/v1/score-sets/{publish_score_set['urn']}/scores?drop_na_columns=true" + f"/api/v1/score-sets/{publish_score_set['urn']}/scores?drop_unused_hgvs_columns=true" ) assert download_scores_csv_response.status_code == 200 download_scores_csv = download_scores_csv_response.text diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py new file mode 100644 index 000000000..14de3a9f6 --- /dev/null +++ b/tests/routers/test_variant.py @@ -0,0 +1,305 @@ +# ruff: noqa: E402 + +import csv +from io import StringIO +from unittest.mock import patch +from urllib.parse import quote + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from sqlalchemy import select + +from tests.helpers.dependency_overrider import DependencyOverrider +from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.score_set import ( + create_seq_score_set_with_mapped_variants, + link_clinvar_control_to_mapped_variant, + publish_score_set, +) + + +def _first_variant_urn(session, score_set_urn): + score_set = session.scalars(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == score_set_urn)).one() + return score_set.variants[0].urn + + +def _csv_path(variant_urn): + """Variant URNs contain a '#', which must be percent-encoded or it is read as a URL fragment.""" + return f"/api/v1/variants/{quote(variant_urn, safe='')}/csv" + + +def _published_score_set_with_mapped_variants(client, session, data_provider, data_files): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + return published + + +class TestGetVariantCsv: + def test_returns_csv_attachment(self, session, data_provider, client, setup_router_db, data_files): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(_csv_path(variant_urn)) + + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/csv") + assert response.headers["content-disposition"] == f'attachment; filename="{variant_urn}.csv"' + + def test_reports_the_requested_variant(self, session, data_provider, client, setup_router_db, data_files): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(_csv_path(variant_urn)) + rows = list(csv.DictReader(StringIO(response.text))) + + assert len(rows) == 1 + assert rows[0]["accession"] == variant_urn + assert rows[0]["score_set.score_set_urn"] == published["urn"] + assert rows[0]["relationship.match_type"] == "exact" + + def test_namespaces_restrict_the_columns(self, session, data_provider, client, setup_router_db, data_files): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(f"{_csv_path(variant_urn)}?namespaces=scores") + + assert response.status_code == 200 + header = response.text.splitlines()[0] + assert "scores.score" in header + assert "gnomad.gnomad_af" not in header + assert "relationship.match_type" not in header + + def test_clinvar_namespace_is_labeled_with_its_release( + self, session, data_provider, client, setup_router_db, data_files + ): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + link_clinvar_control_to_mapped_variant(session, score_set) + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(_csv_path(variant_urn)) + + assert response.status_code == 200 + # The seeded ClinVar control is release 11_2024. + assert "clinvar.2024_11.clinical_significance" in response.text.splitlines()[0] + + @pytest.mark.parametrize( + "namespace", + ["bogus", "clinvar", "clinvar.2024_13", "calibration", "calibration.not-a-urn"], + ) + def test_invalid_namespace_is_rejected( + self, session, data_provider, client, setup_router_db, data_files, namespace + ): + """FastAPI validates the namespace vocabulary from the parameter type, before the handler runs.""" + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(f"{_csv_path(variant_urn)}?namespaces={namespace}") + + assert response.status_code == 422 + errors = response.json()["detail"] + assert any(error["input"] == namespace for error in errors) + + @pytest.mark.parametrize("namespace", ["scores", "clinvar.2024_01"]) + def test_valid_namespace_is_accepted(self, session, data_provider, client, setup_router_db, data_files, namespace): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(f"{_csv_path(variant_urn)}?namespaces={namespace}") + + assert response.status_code == 200 + + def test_namespace_vocabulary_is_published_to_openapi(self, client): + """The generated schema is what the frontend builds its namespace selector from.""" + schema = client.app.openapi()["paths"]["/api/v1/variants/{urn}/csv"]["get"] + namespaces_param = next(param for param in schema["parameters"] if param["name"] == "namespaces") + + item_schema = next( + option["items"] for option in namespaces_param["schema"]["anyOf"] if option.get("type") == "array" + ) + published = {value for option in item_schema["anyOf"] if "enum" in option for value in option["enum"]} + patterns = [option["pattern"] for option in item_schema["anyOf"] if "pattern" in option] + + assert { + "scores", + "scores_custom", + "counts", + "mavedb", + "vep", + "gnomad", + "clingen", + "score_set", + "relationship", + } == published + assert any("clinvar" in pattern for pattern in patterns) + assert any("calibration" in pattern for pattern in patterns) + + def test_unknown_variant_returns_404(self, client, setup_router_db): + response = client.get(_csv_path("urn:mavedb:00000000-a-1#1")) + + assert response.status_code == 404 + assert "not found" in response.json()["detail"] + + def test_private_score_set_is_not_readable_by_other_users( + self, session, data_provider, client, setup_router_db, data_files, extra_user_app_overrides + ): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant_urn = _first_variant_urn(session, score_set["urn"]) + + with DependencyOverrider(extra_user_app_overrides): + response = client.get(_csv_path(variant_urn)) + + assert response.status_code == 404 + + +class TestPrivateCalibrationsAreNotServedOverHttp: + """The lib tests cover the gating logic; this covers the wiring that reaches it. + + The predicate is hand-threaded through four signatures, so the endpoint is the contract worth pinning: + a caller who may read the score set but not the calibration must get NA, even naming the URN outright. + """ + + def _private_calibration(self, session, score_set_urn): + from mavedb.models.score_calibration import ScoreCalibration + + score_set = session.scalars(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == score_set_urn)).one() + calibration = ScoreCalibration( + score_set_id=score_set.id, + urn="urn:mavedb:calibration-99999999-9999-9999-9999-999999999999", + title="Unpublished Calibration", + baseline_score=0.0, + research_use_only=False, + primary=False, + private=True, + calibration_metadata={}, + created_by_id=score_set.created_by_id, + modified_by_id=score_set.modified_by_id, + ) + session.add(calibration) + session.commit() + return calibration.urn + + def test_another_user_naming_the_urn_gets_no_interpretation( + self, session, data_provider, client, setup_router_db, data_files, extra_user_app_overrides + ): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + calibration_urn = self._private_calibration(session, published["urn"]) + namespace = f"calibration.{calibration_urn}" + + with DependencyOverrider(extra_user_app_overrides): + response = client.get( + f"/api/v1/score-sets/{published['urn']}/variants/data?namespaces=scores&namespaces={quote(namespace)}" + ) + + assert response.status_code == 200 + rows = list(csv.DictReader(StringIO(response.text))) + assert rows, "expected variant rows" + assert all(row[f"{namespace}.title"] == "NA" for row in rows) + + def test_another_user_is_not_offered_it_by_discovery( + self, session, data_provider, client, setup_router_db, data_files, extra_user_app_overrides + ): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + calibration_urn = self._private_calibration(session, published["urn"]) + + with DependencyOverrider(extra_user_app_overrides): + response = client.get(f"/api/v1/score-sets/{published['urn']}/csv-namespaces") + + assert response.status_code == 200 + assert f"calibration.{calibration_urn}" not in [entry["namespace"] for entry in response.json()] + + +class TestCsvNamespaceDiscovery: + """The discovery endpoints advertise what a namespace picker should offer.""" + + def test_score_set_namespaces_are_labeled_and_grouped( + self, session, data_provider, client, setup_router_db, data_files + ): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + + response = client.get(f"/api/v1/score-sets/{published['urn']}/csv-namespaces") + + assert response.status_code == 200 + entries = response.json() + by_namespace = {entry["namespace"]: entry for entry in entries} + assert {"scores", "score_set", "vep", "gnomad", "clingen"} <= set(by_namespace) + assert "relationship" not in by_namespace + assert by_namespace["gnomad"]["label"] == "gnomAD allele frequency" + assert by_namespace["gnomad"]["group"] == "annotation" + # Every entry is renderable without the client inventing labels. + assert all(entry["label"] and entry["group"] for entry in entries) + + def test_score_set_detail_response_is_unchanged(self, session, data_provider, client, setup_router_db, data_files): + """Discovery is its own request, so it must not appear on the score-set page's critical path.""" + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + + response = client.get(f"/api/v1/score-sets/{published['urn']}") + + assert response.status_code == 200 + assert "availableCsvNamespaces" not in response.json() + + def test_variant_namespaces_are_labeled_and_grouped( + self, session, data_provider, client, setup_router_db, data_files + ): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(f"/api/v1/variants/{quote(variant_urn, safe='')}/csv-namespaces") + + assert response.status_code == 200 + by_namespace = {entry["namespace"]: entry for entry in response.json()} + # The variant CSV does emit relationship columns, unlike the score-set CSV. + assert "relationship" in by_namespace + assert by_namespace["relationship"]["group"] == "provenance" + + def test_advertised_namespaces_are_accepted_by_the_csv_endpoints( + self, session, data_provider, client, setup_router_db, data_files + ): + """Discovery and validation must agree, or the picker offers options that 422.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + link_clinvar_control_to_mapped_variant(session, score_set) + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + entries = client.get(f"/api/v1/score-sets/{published['urn']}/csv-namespaces").json() + namespaces = [entry["namespace"] for entry in entries] + assert "clinvar.2024_11" in namespaces + + query = "&".join(f"namespaces={quote(ns, safe='')}" for ns in namespaces) + response = client.get(f"/api/v1/score-sets/{published['urn']}/variants/data?{query}") + assert response.status_code == 200 + + variant_urn = _first_variant_urn(session, published["urn"]) + variant_entries = client.get(f"/api/v1/variants/{quote(variant_urn, safe='')}/csv-namespaces").json() + variant_query = "&".join(f"namespaces={quote(entry['namespace'], safe='')}" for entry in variant_entries) + response = client.get(f"{_csv_path(variant_urn)}?{variant_query}") + assert response.status_code == 200 + + def test_unknown_score_set_returns_404(self, client, setup_router_db): + response = client.get("/api/v1/score-sets/urn:mavedb:00000000-a-1/csv-namespaces") + + assert response.status_code == 404 + + def test_unknown_variant_returns_404(self, client, setup_router_db): + response = client.get(f"/api/v1/variants/{quote('urn:mavedb:00000000-a-1#1', safe='')}/csv-namespaces") + + assert response.status_code == 404