From 0f241f313473b599840d3e17bcd02ef4c9795764 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:27:46 +0100 Subject: [PATCH 01/17] feat(dev): og dev build rebuilds the three committed asset tables og dev build cn-intervals, callability and sv-rule-regions read the pinned downloads from a folder and write bin/cn_loci.tsv's interval rows, assets/callability_genes.tsv and assets/sv_rule_regions.tsv, byte for byte as the Python builders did. The GTF is read as bytes with a hand scanner of its attributes, the cut sets are merged once per chromosome and searched by binary search, and og never downloads. The SV rule builder now keeps the first GENCODE record per HGNC id and skips _PAR_Y copies, as the callability builder does; the committed table does not change. Bead: pgp-bs9u.35 --- rust/Cargo.lock | 1 + rust/crates/og-dev/Cargo.toml | 3 +- rust/crates/og-dev/src/build/callability.rs | 628 ++++++++++ rust/crates/og-dev/src/build/cn_intervals.rs | 1162 ++++++++++++++++++ rust/crates/og-dev/src/build/gtf.rs | 133 ++ rust/crates/og-dev/src/build/mod.rs | 298 +++++ rust/crates/og-dev/src/build/sv_rules.rs | 649 ++++++++++ rust/crates/og-dev/src/lib.rs | 6 +- rust/og/src/main.rs | 1 + 9 files changed, 2879 insertions(+), 2 deletions(-) create mode 100644 rust/crates/og-dev/src/build/callability.rs create mode 100644 rust/crates/og-dev/src/build/cn_intervals.rs create mode 100644 rust/crates/og-dev/src/build/gtf.rs create mode 100644 rust/crates/og-dev/src/build/mod.rs create mode 100644 rust/crates/og-dev/src/build/sv_rules.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 4d91a18ca..44bf8f81a 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -666,6 +666,7 @@ version = "0.1.0" dependencies = [ "flate2", "indexmap", + "og-intervals", "og-io", "og-nf", "png", diff --git a/rust/crates/og-dev/Cargo.toml b/rust/crates/og-dev/Cargo.toml index 20bcd7e4c..91e36986b 100644 --- a/rust/crates/og-dev/Cargo.toml +++ b/rust/crates/og-dev/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "og-dev" -description = "og dev: the developer and CI verbs: parity-compare, contract, schema, report-only" +description = "og dev: the developer and CI verbs: parity-compare, contract, schema, report-only, build" version.workspace = true edition.workspace = true license.workspace = true @@ -10,6 +10,7 @@ publish.workspace = true pycompat.workspace = true og-io.workspace = true og-nf.workspace = true +og-intervals.workspace = true flate2.workspace = true indexmap.workspace = true png = "0.18" diff --git a/rust/crates/og-dev/src/build/callability.rs b/rust/crates/og-dev/src/build/callability.rs new file mode 100644 index 000000000..d7ce4dece --- /dev/null +++ b/rust/crates/og-dev/src/build/callability.rs @@ -0,0 +1,628 @@ +//! `og dev build callability`: `assets/callability_genes.tsv`, the genes +//! where short reads are often not placed with confidence, so a "no +//! finding" there needs a reading of its own (step 16b measures each one). +//! +//! A gene is a GENCODE 50 gene with an HGNC id, on chr1-22, X or Y unless it +//! is ALT-only, and protein-coding unless it is a CMRG gene. It is listed +//! with every reason that applies: +//! +//! - `CMRG`: one of GIAB's 273 challenging medically relevant genes, +//! matched by symbol, or by the protein-coding gene that covers most of +//! its CMRG span when GENCODE has renamed it; +//! - `segdup`: half or more of the gene body in a segmental duplication; +//! - `false_duplication`: the gene body overlaps a false duplication of +//! GRCh38 (either copy); +//! - `collapsed`: the gene body overlaps a collapsed duplication; +//! - `ALT-only`: the gene is on an ALT scaffold and nowhere on chr1-22, X, +//! Y or M; +//! +//! and the step that calls it with a method of its own: `parascopy` for the +//! genes of `assets/paralog_loci.tsv`, `pypgx` for the bam genes of +//! `assets/pgx_genes.tsv` callable without ALT contigs, `cn_loci:` +//! for a gene a gene row of `bin/cn_loci.tsv` names; `.` for none. +//! +//! `--work` holds the seven downloads `scripts/ci/build-callability.sh` +//! names. The GTF is read once, line by line; each interval file is merged +//! once per chromosome, so a gene's coverage is a binary search. + +use super::gtf::Record; +use super::{Args, BuildError, CHROMS, Failure, chrom_index, each_line, each_raw_line, int}; +use og_intervals::sets::Set; +use og_io::Utf8; +use std::collections::{HashMap, HashSet}; +use std::fmt::Write as _; +use std::io::Write; +use std::path::Path; + +/// The reasons, in the order a row lists them. +const REASONS: [&str; 5] = [ + "CMRG", + "segdup", + "false_duplication", + "collapsed", + "ALT-only", +]; +const CMRG: u8 = 1; +const SEGDUP: u8 = 1 << 1; +const FALSE_DUP: u8 = 1 << 2; +const COLLAPSED: u8 = 1 << 3; +const ALT_ONLY: u8 = 1 << 4; + +/// A gene of GENCODE on chr1-22, X or Y: the first record of its HGNC id. +struct Gene { + symbol: String, + hgnc: String, + chrom: usize, + start: i64, + end: i64, + coding: bool, +} + +/// What the GTF gives. +#[derive(Default)] +struct Genes { + /// In the GTF's order. + primary: Vec, + by_hgnc: HashMap, + /// Every gene symbol on chr1-22, X, Y or M. + on_primary: HashSet, + /// HGNC id -> (symbol, UCSC ALT scaffold name) of the first + /// protein-coding ALT copy. + alt: HashMap, +} + +/// The ALT scaffolds of NCBI's assembly report: GenBank accession -> UCSC +/// name (GENCODE names them by accession). +fn read_alt(path: &Path) -> Result, BuildError> { + let mut alt = HashMap::new(); + each_line(path, Utf8::Strict, |line| { + let c: Vec<&str> = line.split('\t').collect(); + if !line.starts_with('#') && c.len() >= 10 && c[1] == "alt-scaffold" { + alt.insert(c[4].to_owned(), c[9].to_owned()); + } + Ok(()) + })?; + Ok(alt) +} + +fn read_gtf(path: &Path, alt: &HashMap) -> Result { + let mut g = Genes::default(); + each_raw_line(path, |line| { + let Some(r) = Record::parse(line)? else { + return Ok(()); + }; + if r.feature != b"gene" { + return Ok(()); + } + let chrom = chrom_index(r.seq); + if (chrom.is_some() || r.seq == b"chrM") + && let Some(n) = r.attr("gene_name")? + { + g.on_primary.insert(n.to_owned()); + } + let Some(h) = r.attr("hgnc_id")?.filter(|h| !h.is_empty()) else { + return Ok(()); + }; + let need = |k: &str| { + r.attr(k)?.ok_or_else(|| { + BuildError::Input(format!( + "GTF: a gene with an HGNC id and no {k}: {:?}", + String::from_utf8_lossy(line) + )) + }) + }; + if need("gene_id")?.ends_with("_PAR_Y") { + return Ok(()); + } + let coding = r.attr("gene_type")? == Some("protein_coding"); + match chrom { + Some(chrom) if !g.by_hgnc.contains_key(h) => { + let (start, end) = r.span()?; + g.by_hgnc.insert(h.to_owned(), g.primary.len()); + g.primary.push(Gene { + symbol: need("gene_name")?.to_owned(), + hgnc: h.to_owned(), + chrom, + start, + end, + coding, + }); + } + _ => { + if let Some(name) = std::str::from_utf8(r.seq).ok().and_then(|s| alt.get(s)) + && coding + && !g.alt.contains_key(h) + { + g.alt + .insert(h.to_owned(), (need("gene_name")?.to_owned(), name.clone())); + } + } + } + Ok(()) + })?; + Ok(g) +} + +/// A BED file as a [`Set`] per sequence. +fn read_sets(paths: &[&Path]) -> Result, BuildError> { + let mut rows: HashMap> = HashMap::new(); + for path in paths { + each_line(path, Utf8::Strict, |line| { + let c: Vec<&str> = line.split('\t').collect(); + if c.len() >= 3 + && !(line.starts_with('#') + || line.starts_with("track") + || line.starts_with("browser")) + { + let s = u64::try_from(int(c[1], "BED start")?); + let e = u64::try_from(int(c[2], "BED end")?); + let (Ok(s), Ok(e)) = (s, e) else { + return Err(BuildError::Input(format!( + "{}: a negative position: {line:?}", + path.display() + ))); + }; + rows.entry(c[0].to_owned()).or_default().push((s, e)); + } + Ok(()) + })?; + } + Ok(rows.into_iter().map(|(k, v)| (k, Set::new(v))).collect()) +} + +/// Bases of `[s, e)` on `chrom` inside `sets`. +fn covered(sets: &HashMap, chrom: usize, s: i64, e: i64) -> u64 { + let (s, e) = (u64::try_from(s).unwrap_or(0), u64::try_from(e).unwrap_or(0)); + sets.get(CHROMS[chrom]).map_or(0, |set| set.covered(s, e)) +} + +/// The rows of a tab-separated table after its comment lines, as +/// `csv.DictReader` with `QUOTE_NONE` reads them: the first line is the +/// header; each row is handed to `f` with a cell getter. +fn each_row( + path: &Path, + mut f: impl FnMut(&dyn Fn(&str) -> Result) -> Result<(), BuildError>, +) -> Result<(), BuildError> { + let mut header: Option> = None; + each_line(path, Utf8::Strict, |line| { + if pycompat::text::strip(line, None).is_empty() || line.starts_with('#') { + return Ok(()); + } + let cells: Vec<&str> = line.split('\t').collect(); + let Some(h) = &header else { + header = Some(cells.iter().map(|c| (*c).to_owned()).collect()); + return Ok(()); + }; + let get = |k: &str| { + h.iter() + .rposition(|n| n == k) + .and_then(|i| cells.get(i)) + .map(|c| (*c).to_owned()) + .ok_or_else(|| { + BuildError::Input(format!("{}: no {k} cell in {line:?}", path.display())) + }) + }; + f(&get) + }) +} + +/// Who calls a gene with a method of its own. +struct Owners { + parascopy: HashSet, + pypgx: HashSet, + /// A name a `bin/cn_loci.tsv` gene row reports -> its locus (the first + /// row that names it). + cn: HashMap, +} + +impl Owners { + fn read(repo: &Path) -> Result { + let mut parascopy = HashSet::new(); + each_row(&repo.join("assets/paralog_loci.tsv"), |get| { + parascopy.extend(get("genes")?.split(',').map(str::to_owned)); + Ok(()) + })?; + // A gene step 32 skips on the default no-ALT reference + // (callable_without_alt no) has no caller there, and step 32 + // refuses a BAM with ALT contigs, so it is not pypgx's. + let mut pypgx = HashSet::new(); + each_row(&repo.join("assets/pgx_genes.tsv"), |get| { + if get("mode")? == "bam" && get("callable_without_alt")? != "no" { + pypgx.insert(get("gene")?); + } + Ok(()) + })?; + // A gene row reports the genes it names (its locus, as RHD, or LPA + // for LPA_KIV2, or a target, as HBA1 and HBA2), not every gene whose + // span holds one of its targets. + let mut cn: HashMap = HashMap::new(); + each_row(&repo.join("bin/cn_loci.tsv"), |get| { + if get("kind")? != "gene" { + return Ok(()); + } + let locus = get("locus")?; + let targets = get("targets")?; + let mut names = vec![locus.as_str(), first(&locus, '_')]; + for part in targets.split(';') { + let n = first(part, ':'); + names.extend([n, first(n, '_')]); + } + for n in names { + cn.entry(n.to_owned()).or_insert_with(|| locus.clone()); + } + Ok(()) + })?; + Ok(Owners { + parascopy, + pypgx, + cn, + }) + } + + fn of(&self, symbol: &str) -> String { + if self.parascopy.contains(symbol) { + "parascopy".to_owned() + } else if self.pypgx.contains(symbol) { + "pypgx".to_owned() + } else if let Some(locus) = self.cn.get(symbol) { + format!("cn_loci:{locus}") + } else { + ".".to_owned() + } + } +} + +/// The text before the first `sep`. +fn first(s: &str, sep: char) -> &str { + s.split(sep).next().unwrap_or(s) +} + +/// The reasons of each gene, by HGNC id. +fn reasons(work: &Path, genes: &Genes) -> Result, BuildError> { + let mut why: HashMap = HashMap::new(); + let mut by_symbol: HashMap<&str, usize> = HashMap::new(); + for (i, g) in genes.primary.iter().enumerate() { + by_symbol.insert(&g.symbol, i); + } + let cmrg = work.join("cmrg.bed"); + each_line(&cmrg, Utf8::Strict, |line| { + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 4 { + return Err(BuildError::Input(format!( + "cmrg.bed: a row with under 4 columns: {line:?}" + ))); + } + let (chrom, sym) = (c[0], c[3]); + let (s, e) = (int(c[1], "cmrg.bed start")?, int(c[2], "cmrg.bed end")?); + let i = match by_symbol.get(sym) { + Some(&i) => i, + None => { + // Renamed since CMRG v1.00: the protein-coding gene that + // covers most of its span (the larger HGNC id on a tie). + let best = genes + .primary + .iter() + .enumerate() + .filter(|(_, g)| g.coding && CHROMS[g.chrom] == chrom) + .map(|(i, g)| (e.min(g.end) - s.max(g.start), &g.hgnc, i)) + .max_by(|a, b| (a.0, a.1).cmp(&(b.0, b.1))); + match best { + Some((overlap, _, i)) if overlap > 0 => i, + _ => { + return Err(BuildError::Data(format!( + "CMRG gene {sym} ({chrom}:{s}-{e}) matches no GENCODE 50 gene" + ))); + } + } + } + }; + *why.entry(genes.primary[i].hgnc.clone()).or_default() |= CMRG; + Ok(()) + })?; + let segdups = read_sets(&[&work.join("segdups.bed.gz")])?; + let collapsed = read_sets(&[&work.join("collapsed.bed.gz")])?; + let falsedup = read_sets(&[ + &work.join("falsedup_correct.bed.gz"), + &work.join("falsedup_incorrect.bed.gz"), + ])?; + for g in genes.primary.iter().filter(|g| g.coding) { + let mut bits = 0; + // Half or more: covered >= 0.5 * length, in whole numbers. + if 2 * covered(&segdups, g.chrom, g.start, g.end) + >= u64::try_from(g.end - g.start).unwrap_or(0) + { + bits |= SEGDUP; + } + if covered(&falsedup, g.chrom, g.start, g.end) > 0 { + bits |= FALSE_DUP; + } + if covered(&collapsed, g.chrom, g.start, g.end) > 0 { + bits |= COLLAPSED; + } + if bits != 0 { + *why.entry(g.hgnc.clone()).or_default() |= bits; + } + } + for (h, (sym, _)) in &genes.alt { + if !genes.by_hgnc.contains_key(h) && !genes.on_primary.contains(sym) { + *why.entry(h.clone()).or_default() |= ALT_ONLY; + } + } + Ok(why) +} + +/// A row of the catalog: its sort key (chrom order, start, symbol, HGNC +/// id; ALT-only genes last), its cells and its reasons. +type CatalogRow<'a> = ((usize, i64, &'a str, &'a str), [String; 7], u8); + +/// The catalog's text and the summary line, from the downloads in `work` +/// and the owner tables of `repo`. +pub fn build(work: &Path, repo: &Path) -> Result<(String, String), BuildError> { + let alt = read_alt(&work.join("report.txt"))?; + let genes = read_gtf(&work.join("gencode.gtf.gz"), &alt)?; + let why = reasons(work, &genes)?; + let owners = Owners::read(repo)?; + let mut rows: Vec = Vec::with_capacity(why.len()); + for (h, &bits) in &why { + let (key, chrom, start, end, sym) = match genes.by_hgnc.get(h) { + Some(&i) => { + let g = &genes.primary[i]; + let key = (g.chrom, g.start, g.symbol.as_str(), h.as_str()); + ( + key, + CHROMS[g.chrom].to_owned(), + g.start.to_string(), + g.end.to_string(), + &g.symbol, + ) + } + None => { + let (sym, name) = &genes.alt[h]; + ( + (CHROMS.len(), 0, sym.as_str(), h.as_str()), + name.clone(), + ".".to_owned(), + ".".to_owned(), + sym, + ) + } + }; + let listed: Vec<&str> = REASONS + .iter() + .enumerate() + .filter(|(i, _)| bits & (1 << i) != 0) + .map(|(_, r)| *r) + .collect(); + let cells = [ + sym.clone(), + h.clone(), + chrom, + start, + end, + listed.join(","), + owners.of(sym), + ]; + rows.push((key, cells, bits)); + } + rows.sort_by(|a, b| a.0.cmp(&b.0)); + let mut seen: HashMap<&str, usize> = HashMap::new(); + for r in &rows { + *seen.entry(r.0.2).or_default() += 1; + } + let mut dup: Vec<&str> = seen + .into_iter() + .filter(|&(_, n)| n > 1) + .map(|(s, _)| s) + .collect(); + if !dup.is_empty() { + dup.sort_unstable(); + return Err(BuildError::Data(format!( + "gene symbols listed twice: {}", + dup.join(", ") + ))); + } + let counts: Vec = REASONS + .iter() + .enumerate() + .map(|(i, r)| { + format!( + "{r} {}", + rows.iter().filter(|x| x.2 & (1 << i) != 0).count() + ) + }) + .collect(); + let owned = rows.iter().filter(|r| r.1[6] != ".").count(); + let summary = format!( + "Genes: {} ({}); with an owner: {owned}", + rows.len(), + counts.join(", ") + ); + let mut text = String::from( + "# assets/callability_genes.tsv: the genes where short reads are often not placed with confidence, so a\n\ + # 'no finding' there needs a reading of its own. Step 16b measures each gene on chr1-22, X and Y; the\n\ + # report lists the ones it could not read. Written by scripts/ci/build-callability.sh, which says how the\n\ + # rows and the owners are chosen; rebuild it there, do not edit it by hand. start is 0-based and end\n\ + # 1-based, as in a BED file; an ALT-only gene has its ALT scaffold as chrom and '.' as start and end.\n", + ); + let _ = writeln!(text, "# {summary}."); + text.push_str("gene\thgnc_id\tchrom\tstart\tend\treasons\towner\n"); + for r in &rows { + text.push_str(&r.1.join("\t")); + text.push('\n'); + } + Ok((text, summary)) +} + +pub(super) fn cli(args: &[String], err: &mut dyn Write) -> Result { + let a = Args::parse(args, &["--work", "--repo", "--out"], &[])?; + let (work, repo, out) = (a.need("--work")?, a.need("--repo")?, a.need("--out")?); + let (text, summary) = build(Path::new(work), Path::new(repo))?; + super::write_atomic(Path::new(out), text.as_bytes())?; + let _ = writeln!(err, "{summary}"); + Ok(0) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + /// A work folder and a repo folder with tiny inputs: A is a CMRG gene by + /// symbol, OLD a CMRG gene GENCODE renamed to B, C sits in a segmental + /// duplication, D over a false and a collapsed duplication, E is only on + /// an ALT scaffold, F is on an ALT scaffold and chr1, P is a CMRG gene whose + /// _PAR_Y copy comes first and must not move it to chrY, N is not protein-coding and has no reason. + fn fixture(dir: &Path) -> (std::path::PathBuf, std::path::PathBuf) { + let (work, repo) = (dir.join("work"), dir.join("repo")); + fs::create_dir_all(&work).unwrap(); + fs::create_dir_all(repo.join("assets")).unwrap(); + fs::create_dir_all(repo.join("bin")).unwrap(); + let gene = |seq: &str, s: u32, e: u32, id: &str, ty: &str, name: &str, h: &str| { + let hg = if h.is_empty() { + String::new() + } else { + format!(" hgnc_id \"{h}\";") + }; + format!( + "{seq}\tX\tgene\t{s}\t{e}\t.\t+\t.\tgene_id \"{id}\"; gene_type \"{ty}\"; gene_name \"{name}\";{hg}\n" + ) + }; + let gtf = [ + "##format: gtf\n".to_owned(), + gene("chr1", 101, 200, "G1", "protein_coding", "A", "HGNC:1"), + "chr1\tX\texon\t101\t150\t.\t+\t.\tgene_id \"G1\";\n".to_owned(), + gene("chr1", 1001, 2000, "G2", "protein_coding", "B", "HGNC:2"), + gene("chr2", 101, 1100, "G3", "protein_coding", "C", "HGNC:3"), + gene("chr2", 5001, 6000, "G4", "protein_coding", "D", "HGNC:4"), + gene("chr2", 9001, 9100, "G5", "lncRNA", "N", "HGNC:5"), + gene( + "chrY", + 100_001, + 100_500, + "G6_PAR_Y", + "protein_coding", + "P", + "HGNC:6", + ), + gene( + "chrX", + 100_001, + 100_500, + "G6", + "protein_coding", + "P", + "HGNC:6", + ), + gene("chr1", 3001, 3100, "G7", "protein_coding", "F", "HGNC:7"), + gene("KI1.1", 1, 100, "G8", "protein_coding", "E", "HGNC:8"), + gene("KI1.1", 1, 100, "G9", "protein_coding", "F", "HGNC:9"), + gene("chrM", 1, 50, "G10", "protein_coding", "MT", ""), + ] + .concat(); + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + gz.write_all(gtf.as_bytes()).unwrap(); + fs::write(work.join("gencode.gtf.gz"), gz.finish().unwrap()).unwrap(); + fs::write( + work.join("report.txt"), + "# report\nKI1\talt-scaffold\t1\tc\tKI1.1\t=\tx\ty\tz\tchr1_KI1v1_alt\r\n", + ) + .unwrap(); + fs::write( + work.join("cmrg.bed"), + "chr1\t100\t200\tA\nchr1\t900\t1500\tOLD\nchrX\t100000\t100500\tP\n", + ) + .unwrap(); + fs::write( + work.join("segdups.bed.gz"), + "chr2\t100\t400\nchr2\t350\t700\n", + ) + .unwrap(); + fs::write(work.join("falsedup_correct.bed.gz"), "chr2\t5999\t6001\n").unwrap(); + fs::write(work.join("falsedup_incorrect.bed.gz"), "").unwrap(); + fs::write(work.join("collapsed.bed.gz"), "#c\nchr2\t5000\t5001\n").unwrap(); + fs::write( + repo.join("assets/paralog_loci.tsv"), + "# x\nlocus\tgenes\nl\tC,Z\n", + ) + .unwrap(); + fs::write( + repo.join("assets/pgx_genes.tsv"), + "gene\tmode\tcallable_without_alt\nD\tbam\tyes\nA\tbam\tno\n", + ) + .unwrap(); + fs::write( + repo.join("bin/cn_loci.tsv"), + "locus\tkind\ttargets\nA_x\tgene\tq_t1:1-2\nY\tinterval\tA:1-2\n", + ) + .unwrap(); + (work, repo) + } + + #[test] + fn the_tiny_catalog_has_every_reason_and_owner() { + let dir = std::env::temp_dir().join(format!("og-callability-{}", std::process::id())); + let (work, repo) = fixture(&dir); + let (text, summary) = build(&work, &repo).unwrap(); + fs::remove_dir_all(&dir).unwrap(); + let body: Vec<&str> = text.lines().filter(|l| !l.starts_with('#')).collect(); + assert_eq!( + body, + [ + "gene\thgnc_id\tchrom\tstart\tend\treasons\towner", + "A\tHGNC:1\tchr1\t100\t200\tCMRG\tcn_loci:A_x", + "B\tHGNC:2\tchr1\t1000\t2000\tCMRG\t.", + "C\tHGNC:3\tchr2\t100\t1100\tsegdup\tparascopy", + "D\tHGNC:4\tchr2\t5000\t6000\tfalse_duplication,collapsed\tpypgx", + "P\tHGNC:6\tchrX\t100000\t100500\tCMRG\t.", + "E\tHGNC:8\tchr1_KI1v1_alt\t.\t.\tALT-only\t.", + ] + ); + assert_eq!( + summary, + "Genes: 6 (CMRG 3, segdup 1, false_duplication 1, collapsed 1, ALT-only 1); with an owner: 3" + ); + assert!(text.contains(&format!("\n# {summary}.\n"))); + } + + #[test] + fn a_cmrg_gene_with_no_gencode_gene_stops_the_build() { + let dir = std::env::temp_dir().join(format!("og-callability-cmrg-{}", std::process::id())); + let (work, repo) = fixture(&dir); + fs::write(work.join("cmrg.bed"), "chr3\t1\t2\tGONE\n").unwrap(); + let e = build(&work, &repo).err().unwrap(); + fs::remove_dir_all(&dir).unwrap(); + assert_eq!( + e.to_string(), + "CMRG gene GONE (chr3:1-2) matches no GENCODE 50 gene" + ); + assert!(matches!(e, BuildError::Data(_))); + } + + #[test] + fn a_symbol_listed_twice_stops_the_build() { + let dir = std::env::temp_dir().join(format!("og-callability-dup-{}", std::process::id())); + let (work, repo) = fixture(&dir); + // Two genes named A, both CMRG (the second by its span). + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + gz.write_all(b"chr1\tX\tgene\t101\t200\t.\t+\t.\tgene_id \"G1\"; gene_type \"protein_coding\"; gene_name \"A\"; hgnc_id \"HGNC:1\";\nchr1\tX\tgene\t901\t1500\t.\t+\t.\tgene_id \"G2\"; gene_type \"protein_coding\"; gene_name \"A\"; hgnc_id \"HGNC:2\";\n").unwrap(); + fs::write(work.join("gencode.gtf.gz"), gz.finish().unwrap()).unwrap(); + fs::write( + work.join("cmrg.bed"), + "chr1\t100\t200\tX1\nchr1\t900\t1500\tX2\n", + ) + .unwrap(); + let e = build(&work, &repo).err().unwrap(); + fs::remove_dir_all(&dir).unwrap(); + assert_eq!(e.to_string(), "gene symbols listed twice: A"); + } + + #[test] + fn coverage_is_union_coverage() { + let sets = HashMap::from([( + "chr2".to_owned(), + Set::new([(100, 400), (350, 700), (900, 950)]), + )]); + assert_eq!(covered(&sets, 1, 100, 1100), 650); + assert_eq!(covered(&sets, 0, 100, 1100), 0); + } +} diff --git a/rust/crates/og-dev/src/build/cn_intervals.rs b/rust/crates/og-dev/src/build/cn_intervals.rs new file mode 100644 index 000000000..8624daa9b --- /dev/null +++ b/rust/crates/og-dev/src/build/cn_intervals.rs @@ -0,0 +1,1162 @@ +//! `og dev build cn-intervals`: the interval rows of `bin/cn_loci.tsv`, +//! the copy-number checks step 35 runs with one `samtools bedcov` pass +//! (`og cn intervals`). +//! +//! The rows are: +//! +//! - every region row of ClinGen Dosage Sensitivity's recurrent CNV BED, +//! version 2.1, GRCh38 (`CLINGEN_RECURRENT_CNV_URL` in `versions.env`, +//! CC0). Its breakpoint and LCR rows (itemRgb `0,0,0`) are the segmental +//! duplications that bound the regions: they are cut out of the targets +//! and kept out of the controls. Its chrX rows are spelled `chrx` and are +//! read as chrX; any other name that is not a GRCh38 chromosome stops the +//! build; +//! - the rows the BED lacks (`HAND_ROWS`): two ClinGen curations (1p36, +//! Xq28 MECP2), five carrier deletions at their ClinVar or HbVar GRCh38 +//! spans, and the Y chromosome's AZF regions with two single-copy Y +//! controls. +//! +//! Targets are cut to pieces of at most 500 kb and lose every stretch inside +//! a breakpoint or LCR row, a GIAB v3.6 segmental duplication, an assembly +//! gap (GIAB's gaps with 15 kb on each side), a satellite or a region of a +//! model the pinned Parascopy data holds; the AZF targets keep their +//! segmental duplications, since the DAZ copies are one. Controls are two +//! 100 kb windows, one on each side of the cluster of ClinGen rows the +//! region belongs to, the nearest that stay clear of all of that and of +//! every target, within 3 Mb; a side with none (a chromosome end) is left +//! out. +//! +//! Every cut set is merged once per chromosome, so a subtraction or a +//! "clear of the cuts" test is a binary search. + +use super::{Args, BuildError, CHROMS, Failure, chrom_index, each_line, input_error, int}; +use og_io::Utf8; +use std::io::Write; +use std::path::Path; + +/// The line after which the catalog's rows are this builder's. +pub const MARKER: &str = + "# --- interval rows: written by scripts/ci/build-cn-intervals.sh; do not edit by hand ---"; +/// How a marker line starts, whatever builder it names. +const MARKER_START: &[u8] = b"# --- interval rows: "; + +const REGION_RGB: &str = "255,111,0"; +const MASK_RGB: &str = "0,0,0"; +/// GRCh38 chromosome lengths (the no-ALT analysis set's `.fai`), in +/// [`CHROMS`] order. +const LENGTHS: [i64; 24] = [ + 248_956_422, + 242_193_529, + 198_295_559, + 190_214_555, + 181_538_259, + 170_805_979, + 159_345_973, + 145_138_636, + 138_394_717, + 133_797_422, + 135_086_622, + 133_275_309, + 114_364_328, + 107_043_718, + 101_991_189, + 90_338_345, + 83_257_441, + 80_373_285, + 58_617_616, + 64_444_167, + 46_709_983, + 50_818_468, + 156_040_895, + 57_227_415, +]; +const PIECE: i64 = 500_000; +const FLANK: i64 = 100_000; +const GAP: i64 = 50_000; +const SEARCH: i64 = 3_000_000; +const STEP: i64 = 10_000; +const MIN_PIECE: i64 = 500; +const CHR_Y: usize = 23; + +/// A 0-based, half-open stretch. +type Span = (i64, i64); + +/// How a row's target depth is read. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Depth { + /// Depth pooled over the targets. + Pooled, + /// The targets' depths added up (copies of one sequence). + Sum, +} + +impl Depth { + fn name(self) -> &'static str { + match self { + Depth::Pooled => "pooled", + Depth::Sum => "sum", + } + } +} + +/// A row the BED lacks. +struct Hand { + locus: &'static str, + chrom: usize, + /// (label, 0-based start, end); the first label names the row. + targets: &'static [(&'static str, i64, i64)], + /// (name, start, end, label), or `None` for flanks. + controls: Option<&'static [(&'static str, i64, i64, &'static str)]>, + depth: Depth, + /// Whether segmental duplications are cut out of the targets. + cut_segdups: bool, + pmids: &'static str, +} + +const Y_CONTROLS: &[(&str, i64, i64, &str)] = &[ + ( + "ZFY", + 2_935_200, + 2_982_579, + "ZFY, single-copy control on Yp", + ), + ( + "NLGN4Y", + 14_522_572, + 14_845_654, + "NLGN4Y, single-copy control on Yq", + ), +]; + +/// The rows the ClinGen BED lacks. 1p36 (ISCA-37434) and Xq28 (ISCA-46304) +/// from ClinGen_region_curation_list_GRCh38.tsv as read on 2026-11-04; +/// del(GJB6-D13S1830), CFTRdele2,3 and the 621 bp HBB deletion at their +/// ClinVar GRCh38 spans; the 619 bp HBB deletion (HbVar 979, Orkin 1979) +/// and the Sicilian (delta-beta)0 deletion (HbVar 1035, Henthorn 1990) at +/// their HbVar spans, placed on NC_000011.10 by 50 bp on each side of the +/// NG_000007.3 breakpoints on 2026-11-04; the AZF regions by the single-copy +/// genes each deletion removes (AZFc as the four DAZ copies added up), gene +/// spans from Ensembl's REST lookup, checked on 2026-11-04. The alpha-globin +/// --SEA deletion is read by step 35's HBA table, not here. +const HAND_ROWS: &[Hand] = &[ + Hand { + locus: "1p36_terminal", + chrom: 0, + targets: &[( + "1p36 terminal region (includes GABRD), ClinGen ISCA-37434", + 898_702, + 6_229_913, + )], + controls: None, + depth: Depth::Pooled, + cut_segdups: true, + pmids: ".", + }, + Hand { + locus: "Xq28_MECP2", + chrom: 22, + targets: &[( + "Xq28 region (includes MECP2), ClinGen ISCA-46304: the MECP2 duplication", + 154_008_528, + 154_110_279, + )], + controls: None, + depth: Depth::Pooled, + cut_segdups: true, + pmids: ".", + }, + Hand { + locus: "GJB6_D13S1830", + chrom: 12, + targets: &[("del(GJB6-D13S1830), ClinVar 5546", 20_223_037, 20_531_806)], + controls: None, + depth: Depth::Pooled, + cut_segdups: true, + pmids: "11807148", + }, + Hand { + locus: "CFTR_dele2_3", + chrom: 6, + targets: &[( + "CFTRdele2,3 (21 kb), ClinVar 66105", + 117_498_312, + 117_519_392, + )], + controls: None, + depth: Depth::Pooled, + cut_segdups: true, + pmids: ".", + }, + Hand { + locus: "HBB_exon3_del", + chrom: 10, + targets: &[( + "HBB deletion of exon 3 (621 bp), ClinVar 3769040", + 5_225_254, + 5_225_875, + )], + controls: None, + depth: Depth::Pooled, + cut_segdups: true, + pmids: ".", + }, + Hand { + locus: "HBB_619bp_del", + chrom: 10, + targets: &[( + "HBB 619 bp deletion (South Asian), HbVar 979", + 5_225_388, + 5_226_007, + )], + controls: None, + depth: Depth::Pooled, + cut_segdups: true, + pmids: "287080", + }, + Hand { + locus: "HBD_HBB_Sicilian_del", + chrom: 10, + targets: &[( + "Sicilian (delta-beta)0 deletion (13.4 kb, HBD to HBB 3'), HbVar 1035", + 5_219_877, + 5_233_280, + )], + controls: None, + depth: Depth::Pooled, + cut_segdups: true, + pmids: "2307466", + }, + Hand { + locus: "AZFa", + chrom: CHR_Y, + targets: &[ + ("AZFa (USP9Y and DDX3Y)", 12_537_649, 12_860_852), + ("DDX3Y", 12_904_107, 12_920_478), + ], + controls: Some(Y_CONTROLS), + depth: Depth::Pooled, + cut_segdups: false, + pmids: "37674303", + }, + Hand { + locus: "AZFb", + chrom: CHR_Y, + targets: &[ + ("AZFb (KDM5D and EIF1AY)", 19_702_297, 19_745_148), + ("EIF1AY", 20_575_693, 20_593_154), + ], + controls: Some(Y_CONTROLS), + depth: Depth::Pooled, + cut_segdups: false, + pmids: "37674303", + }, + Hand { + locus: "AZFc_DAZ", + chrom: CHR_Y, + targets: &[ + ( + "AZFc (DAZ1 to DAZ4 added up; gr/gr and b2/b3 leave two)", + 23_129_354, + 23_199_010, + ), + ("DAZ2", 23_219_446, 23_291_356), + ("DAZ3", 24_763_068, 24_813_492), + ("DAZ4", 24_833_842, 24_907_040), + ], + controls: Some(Y_CONTROLS), + depth: Depth::Sum, + cut_segdups: false, + pmids: "37674303", + }, +]; + +/// A region row of the ClinGen BED. +#[derive(Debug)] +struct Region { + chrom: usize, + start: i64, + end: i64, + name: String, +} + +/// The stretches cut out of targets and controls, per chromosome. +#[derive(Default, Clone)] +struct Cuts { + /// ClinGen breakpoint and LCR rows. + mask: Vec, + segdup: Vec, + gap: Vec, + satellite: Vec, + parascopy: Vec, +} + +/// The downloads, read. +pub struct Inputs { + regions: Vec, + cuts: Vec, +} + +/// `chrx` and `chry` (the ClinGen BED's spelling) as chrX and chrY; any +/// other name that is not a GRCh38 chromosome is an error. +fn norm_chrom(c: &str) -> Result { + let fixed = match c { + "chrx" => "chrX", + "chry" => "chrY", + _ => c, + }; + chrom_index(fixed).ok_or_else(|| { + BuildError::Input(format!( + "chromosome {} is not a GRCh38 chromosome name", + py_repr(c) + )) + }) +} + +fn py_repr(s: &str) -> String { + pycompat::value::py_repr(&pycompat::json::Value::Str(s.to_owned())) + .unwrap_or_else(|_| format!("'{s}'")) +} + +/// The region rows and breakpoint rows of the ClinGen BED's lines. +fn read_clingen<'a>( + lines: impl IntoIterator, + cuts: &mut [Cuts], +) -> Result, BuildError> { + let mut regions = Vec::new(); + for line in lines { + let strip = pycompat::text::strip; + if strip(line, None).is_empty() + || line.starts_with('#') + || line.starts_with("track") + || line.starts_with("browser") + { + continue; + } + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 9 { + return Err(BuildError::Input(format!( + "ClinGen BED: a row with {} columns: {}", + c.len(), + py_repr(strip(line, None)) + ))); + } + let chrom = norm_chrom(c[0])?; + let (start, end) = ( + int(c[1], "ClinGen BED start")?, + int(c[2], "ClinGen BED end")?, + ); + let (name, rgb) = (strip(c[3], None), strip(c[8], None)); + if end <= start { + return Err(BuildError::Input(format!( + "ClinGen BED: {name}: empty span" + ))); + } + match rgb { + REGION_RGB => regions.push(Region { + chrom, + start, + end, + name: name.to_owned(), + }), + MASK_RGB => cuts[chrom].mask.push((start, end)), + _ => { + return Err(BuildError::Input(format!( + "ClinGen BED: {name}: itemRgb {} is neither a region nor a breakpoint row", + py_repr(rgb) + ))); + } + } + } + if regions.is_empty() { + return Err(BuildError::Input("ClinGen BED: no region row".to_owned())); + } + Ok(regions) +} + +/// The rows of a BED on chr1-22, X and Y, per chromosome. +fn read_bed( + path: &Path, + put: impl Fn(&mut Cuts) -> &mut Vec, + cuts: &mut [Cuts], +) -> Result<(), BuildError> { + each_line(path, Utf8::Strict, |line| { + if line.starts_with('#') || pycompat::text::strip(line, None).is_empty() { + return Ok(()); + } + let mut c = line.split('\t'); + let (seq, s, e) = (c.next(), c.next(), c.next()); + let (Some(seq), Some(s), Some(e)) = (seq, s, e) else { + return Err(BuildError::Input(format!( + "{}: a row with under 3 columns", + path.display() + ))); + }; + if let Some(i) = chrom_index(seq) { + let span = (int(s, "BED start")?, int(e, "BED end")?); + put(&mut cuts[i]).push(span); + } + Ok(()) + }) +} + +/// The regions of Parascopy's models: `NAME CHROM:START-END ...`, 1-based. +fn read_parascopy(path: &Path, cuts: &mut [Cuts]) -> Result<(), BuildError> { + each_line(path, Utf8::Strict, |line| { + if line.starts_with('#') || pycompat::text::strip(line, None).is_empty() { + return Ok(()); + } + let bad = || { + BuildError::Input(format!( + "{}: not NAME CHROM:START-END DETAIL: {line:?}", + path.display() + )) + }; + let f = pycompat::text::split(line, None, None); + let [_, reg, _] = f[..] else { + return Err(bad()); + }; + let (c, span) = reg.split_once(':').ok_or_else(bad)?; + let (s, e) = span.split_once('-').ok_or_else(bad)?; + if let Some(i) = chrom_index(c) { + cuts[i] + .parascopy + .push((int(s, "Parascopy start")? - 1, int(e, "Parascopy end")?)); + } + Ok(()) + }) +} + +/// Sorted, with overlapping and touching spans joined. +fn merge(mut spans: Vec) -> Vec { + spans.sort_unstable(); + let mut out: Vec = Vec::with_capacity(spans.len()); + for (s, e) in spans { + match out.last_mut() { + Some(last) if s <= last.1 => last.1 = last.1.max(e), + _ => out.push((s, e)), + } + } + out +} + +/// The parts of `[s, e)` outside the merged `cuts`. +fn subtract(s: i64, e: i64, cuts: &[Span]) -> Vec { + let mut out = Vec::new(); + let mut pos = s; + let first = cuts.partition_point(|c| c.1 <= s); + for &(cs, ce) in &cuts[first..] { + if cs >= e { + break; + } + if cs > pos { + out.push((pos, cs)); + } + pos = pos.max(ce); + } + if pos < e { + out.push((pos, e)); + } + out +} + +/// Each span of `MIN_PIECE` bp or more cut into equal pieces of at most +/// `PIECE` bp. +fn pieces(spans: &[Span]) -> Vec { + let mut out = Vec::new(); + for &(s, e) in spans { + let len = e - s; + if len < MIN_PIECE { + continue; + } + // Ceiling divisions of positive numbers. + let n = (len + PIECE - 1) / PIECE; + let step = (len + n - 1) / n; + out.extend((0..n).map(|i| (s + i * step, e.min(s + (i + 1) * step)))); + } + out +} + +/// Whether `[s, e)` touches none of the merged `cuts`. +fn clear(s: i64, e: i64, cuts: &[Span]) -> bool { + let i = cuts.partition_point(|c| c.1 <= s); + cuts.get(i).is_none_or(|c| c.0 >= e) +} + +/// The nearest `FLANK` window `GAP` beyond `edge` on the left (`left`) or +/// right, clear of `cuts`, within `SEARCH`. +fn flank(chrom: usize, edge: i64, left: bool, cuts: &[Span]) -> Option { + for k in 0..=SEARCH / STEP { + let (s, e) = if left { + let e = edge - GAP - k * STEP; + (e - FLANK, e) + } else { + let s = edge + GAP + k * STEP; + (s, s + FLANK) + }; + if s < 0 || e > LENGTHS[chrom] { + return None; + } + if clear(s, e, cuts) { + return Some((s, e)); + } + } + None +} + +/// The row id of a ClinGen region name: its band, then the words of its +/// parentheses that are not an "includes" note. +fn locus_id(name: &str) -> String { + let band = pycompat::text::split(name, None, None) + .first() + .copied() + .unwrap_or_default(); + let b = name.as_bytes(); + let mut groups: Vec<&str> = Vec::new(); + let mut i = 0; + while i < b.len() { + if b[i] == b'(' { + let j = b[i + 1..] + .iter() + .position(|&c| c == b'(' || c == b')') + .map(|k| i + 1 + k); + if let Some(j) = j + && b[j] == b')' + { + let g = &name[i + 1..j]; + if !pycompat::text::strip(g, None).starts_with("includes") { + groups.push(g); + } + i = j + 1; + continue; + } + } + i += 1; + } + let mut words = String::new(); + let mut in_run = false; + for ch in groups.join(" ").chars() { + if ch.is_ascii_alphanumeric() || ch == '.' || ch == '-' { + words.push(ch); + in_run = false; + } else if !in_run { + words.push('_'); + in_run = true; + } + } + let words = words.trim_matches('_'); + if words.is_empty() { + band.to_owned() + } else { + format!("{band}_{words}") + } +} + +/// A name made safe for a target or control cell. +fn label(text: &str) -> String { + let t: String = text + .chars() + .map(|c| { + if matches!(c, ';' | '\t' | '\n') { + ',' + } else { + c + } + }) + .collect(); + pycompat::text::strip(&t, None).to_owned() +} + +/// A control window: (name, start, end, label). +type Control = (String, i64, i64, String); + +/// The catalog rows, as Python wrote them, in their order. +struct Builder<'a> { + inputs: &'a Inputs, + /// Per chromosome, the ClinGen rows that touch or overlap, merged + /// (each row's end taken one base further). + clusters: Vec>, + /// Per chromosome, every cut (with and without segmental duplications) + /// and the cuts a control keeps out of. + cuts_all: Vec>, + cuts_no_segdups: Vec>, + flank_cuts: Vec>, + rows: Vec, + seen: std::collections::HashMap, +} + +impl<'a> Builder<'a> { + fn new(inputs: &'a Inputs) -> Builder<'a> { + let n = CHROMS.len(); + let mut clusters = vec![Vec::new(); n]; + let mut flank_cuts = vec![Vec::new(); n]; + for r in &inputs.regions { + clusters[r.chrom].push((r.start, r.end + 1)); + flank_cuts[r.chrom].push((r.start, r.end)); + } + for h in HAND_ROWS { + flank_cuts[h.chrom].extend(h.targets.iter().map(|&(_, s, e)| (s, e))); + } + let mut cuts_all = Vec::with_capacity(n); + let mut cuts_no_segdups = Vec::with_capacity(n); + for (i, c) in inputs.cuts.iter().enumerate() { + clusters[i].extend(c.mask.iter().map(|&(s, e)| (s, e + 1))); + let all: Vec = [&c.mask, &c.segdup, &c.gap, &c.satellite, &c.parascopy] + .into_iter() + .flatten() + .copied() + .collect(); + flank_cuts[i].extend(&all); + cuts_all.push(merge(all)); + cuts_no_segdups.push(merge( + [&c.mask, &c.gap, &c.parascopy] + .into_iter() + .flatten() + .copied() + .collect(), + )); + } + Builder { + inputs, + clusters: clusters.into_iter().map(merge).collect(), + cuts_all, + cuts_no_segdups, + flank_cuts: flank_cuts.into_iter().map(merge).collect(), + rows: Vec::new(), + seen: std::collections::HashMap::new(), + } + } + + /// The two flank controls of the cluster that holds `[s, e]`. + fn flanks(&self, chrom: usize, s: i64, e: i64) -> Result, BuildError> { + let (cs, ce) = self.clusters[chrom] + .iter() + .copied() + .find(|&(a, b)| a <= s && e <= b) + .unwrap_or((s, e)); + let mut out = Vec::new(); + for (left, edge, n) in [(true, cs, "L"), (false, ce, "R")] { + if let Some((a, b)) = flank(chrom, edge, left, &self.flank_cuts[chrom]) { + let side = if left { "left" } else { "right" }; + out.push((n.to_owned(), a, b, format!("flank {side} of the region"))); + } + } + if out.is_empty() { + return Err(BuildError::Input(format!( + "{}:{s}-{e}: no flank clear of the masks within {} Mb", + CHROMS[chrom], + SEARCH / 1_000_000 + ))); + } + Ok(out) + } + + fn add( + &mut self, + locus: &str, + chrom: usize, + targets: &[(i64, i64, String)], + controls: &[Control], + depth: Depth, + pmids: &str, + ) -> Result<(), BuildError> { + if targets.is_empty() { + return Err(BuildError::Input(format!( + "{locus}: no target stretch is left outside the masks" + ))); + } + let n = self.seen.entry(locus.to_owned()).or_insert(0); + *n += 1; + let locus = if *n > 1 { + format!("{locus}_{n}") + } else { + locus.to_owned() + }; + let tg: Vec = targets + .iter() + .enumerate() + .map(|(i, (s, e, lab))| { + let tail = if lab.is_empty() { + String::new() + } else { + format!(":{lab}") + }; + format!("{locus}_t{}:{s}-{e}{tail}", i + 1) + }) + .collect(); + let ct: Vec = controls + .iter() + .map(|(n, s, e, lab)| format!("{locus}_{n}:{s}-{e}:{lab}")) + .collect(); + let y = chrom == CHR_Y; + let cells = [ + locus.as_str(), + "interval", + CHROMS[chrom], + &tg.join(";"), + &ct.join(";"), + ".", + depth.name(), + if y { "pooled" } else { "flanks" }, + "2", + "0", + ".", + ".", + ".", + if y { "4" } else { "8" }, + if y { "." } else { "1.5" }, + "round:0.3", + ".", + ".", + "bedcov", + "cn-depth", + "cnvpytor:cnvs", + pmids, + ]; + self.rows.push(cells.join("\t")); + Ok(()) + } + + /// Targets with the first piece labelled. + fn labelled(segs: &[Span], name: &str) -> Vec<(i64, i64, String)> { + segs.iter() + .enumerate() + .map(|(i, &(a, b))| (a, b, if i == 0 { label(name) } else { String::new() })) + .collect() + } + + fn build(mut self) -> Result, BuildError> { + let mut regions: Vec<&Region> = self.inputs.regions.iter().collect(); + regions.sort_by_key(|r| (r.chrom, r.start, r.end)); + for r in regions { + let segs = pieces(&subtract(r.start, r.end, &self.cuts_all[r.chrom])); + let targets = Self::labelled(&segs, &r.name); + let controls = self.flanks(r.chrom, r.start, r.end)?; + self.add( + &locus_id(&r.name), + r.chrom, + &targets, + &controls, + Depth::Pooled, + ".", + )?; + } + for h in HAND_ROWS { + let cuts = if h.cut_segdups { + &self.cuts_all[h.chrom] + } else { + &self.cuts_no_segdups[h.chrom] + }; + let mut targets = Vec::new(); + for &(name, s, e) in h.targets { + let segs = match h.depth { + Depth::Pooled => pieces(&subtract(s, e, cuts)), + Depth::Sum => vec![(s, e)], + }; + if h.depth == Depth::Sum + && !clear(s, e, &merge(self.inputs.cuts[h.chrom].parascopy.clone())) + { + return Err(BuildError::Input(format!( + "{}: {name} overlaps a Parascopy model", + h.locus + ))); + } + targets.extend(Self::labelled(&segs, name)); + } + let controls = match h.controls { + None => self.flanks(h.chrom, h.targets[0].1, h.targets[h.targets.len() - 1].2)?, + Some(c) => c + .iter() + .enumerate() + .map(|(i, &(_, s, e, lab))| (format!("c{}", i + 1), s, e, lab.to_owned())) + .collect(), + }; + self.add(h.locus, h.chrom, &targets, &controls, h.depth, h.pmids)?; + } + Ok(self.rows) + } +} + +impl Inputs { + /// The downloads: the ClinGen BED's lines, the three GIAB BEDs and the + /// Parascopy model list. + pub fn read( + clingen: &Path, + segdups: &Path, + gaps: &Path, + satellites: &Path, + parascopy: &Path, + ) -> Result { + let mut cuts = vec![Cuts::default(); CHROMS.len()]; + let lines: Vec = super::lines(clingen, Utf8::Replace)? + .collect::>() + .map_err(|e| input_error(clingen, &e))?; + let regions = read_clingen(lines.iter().map(String::as_str), &mut cuts)?; + read_bed(segdups, |c| &mut c.segdup, &mut cuts)?; + read_bed(gaps, |c| &mut c.gap, &mut cuts)?; + read_bed(satellites, |c| &mut c.satellite, &mut cuts)?; + read_parascopy(parascopy, &mut cuts)?; + Ok(Inputs { regions, cuts }) + } + + /// The interval rows, without line endings, marker excluded. + pub fn rows(&self) -> Result, BuildError> { + Builder::new(self).build() + } +} + +/// The catalog with its rows from the marker line on replaced: the lines +/// before a marker line are kept (all of them when there is none). +fn splice(catalog: &[u8], rows: &[String]) -> Vec { + let mut out = Vec::with_capacity(catalog.len()); + for line in catalog.split_inclusive(|&c| c == b'\n') { + if line.starts_with(MARKER_START) && line.ends_with(b"\n") { + break; + } + out.extend_from_slice(line); + } + for line in std::iter::once(MARKER).chain(rows.iter().map(String::as_str)) { + out.extend_from_slice(line.as_bytes()); + out.push(b'\n'); + } + out +} + +/// How two catalogs differ: (lines only in `old`, lines only in `new`), +/// each line counted as often as it appears. +fn line_diff(old: &[u8], new: &[u8]) -> (usize, usize) { + let mut count: std::collections::HashMap<&[u8], i64> = std::collections::HashMap::new(); + for l in old.split_inclusive(|&c| c == b'\n') { + *count.entry(l).or_default() += 1; + } + for l in new.split_inclusive(|&c| c == b'\n') { + *count.entry(l).or_default() -= 1; + } + let only_old = count + .values() + .filter(|&&v| v > 0) + .map(|&v| v as usize) + .sum(); + let only_new = count + .values() + .filter(|&&v| v < 0) + .map(|&v| (-v) as usize) + .sum(); + (only_old, only_new) +} + +pub(super) fn cli( + args: &[String], + out: &mut dyn Write, + err: &mut dyn Write, +) -> Result { + let a = Args::parse( + args, + &[ + "--clingen", + "--segdups", + "--gaps", + "--satellites", + "--parascopy", + "--catalog", + ], + &["--check", "--stdout"], + )?; + let catalog = a.get("--catalog"); + if catalog.is_some() == a.has("--stdout") { + return Err(Failure::Usage( + "give one of --catalog FILE and --stdout".to_owned(), + )); + } + if a.has("--check") && catalog.is_none() { + return Err(Failure::Usage("--check needs --catalog".to_owned())); + } + let p = |f: &str| a.need(f).map(Path::new); + let inputs = Inputs::read( + p("--clingen")?, + p("--segdups")?, + p("--gaps")?, + p("--satellites")?, + p("--parascopy")?, + )?; + let rows = inputs.rows()?; + let Some(catalog) = catalog else { + let mut text = String::new(); + for line in std::iter::once(MARKER).chain(rows.iter().map(String::as_str)) { + text.push_str(line); + text.push('\n'); + } + let _ = out.write_all(text.as_bytes()); + return Ok(0); + }; + let path = Path::new(catalog); + let old = std::fs::read(path).map_err(|e| input_error(path, &e))?; + let new = splice(&old, &rows); + if a.has("--check") { + if new != old { + let (only_old, only_new) = line_diff(&old, &new); + let how = if only_old == 0 && only_new == 0 { + "the same lines in another order".to_owned() + } else { + format!("{only_old} line(s) only in the file, {only_new} only in the rebuild") + }; + let _ = writeln!( + err, + "{catalog} is not what the pinned downloads give ({how}); rerun scripts/ci/build-cn-intervals.sh" + ); + return Ok(1); + } + let _ = writeln!( + out, + "{catalog}: the {} interval rows are what the pinned downloads give", + rows.len() + ); + return Ok(0); + } + super::write_atomic(path, &new)?; + let _ = writeln!(out, "{catalog}: {} interval rows written", rows.len()); + Ok(0) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A small ClinGen BED: the 22q11.2 A-D region between its breakpoint + /// rows, and the Xp22.31 region with its breakpoint, spelled `chrx`. + const SMALL_BED: &str = "track name=\"test\"\n#chrom\tstart\tend\tname\tscore\tstrand\tthickStart\tthickEnd\titemRgb\nchr22\t18159724\t18924717\tBP-A\t0\t.\t18159724\t18924717\t0,0,0\nchr22\t18924718\t21111383\t22q11.2 recurrent (DGS) region (proximal, A-D) (includes TBX1)\t0\t.\t18924718\t21111383\t255,111,0\nchr22\t21111384\t21562827\tBP-D\t0\t.\t21111384\t21562827\t0,0,0\nchrx\t6527561\t6537770\tXp22.31 distal breakpoint\t0\t.\t6527561\t6537770\t0,0,0\nchrx\t6537771\t8156913\tXp22.31 recurrent region (includes STS)\t0\t.\t6537771\t8156913\t255,111,0\n"; + + fn small(bed: &str) -> Result, BuildError> { + let mut cuts = vec![Cuts::default(); CHROMS.len()]; + let regions = read_clingen(bed.lines(), &mut cuts)?; + Inputs { regions, cuts }.rows() + } + + fn cell(row: &str, i: usize) -> &str { + row.split('\t').nth(i).unwrap() + } + + /// (start, end) of each target of a row. + fn spans(row: &str) -> Vec { + cell(row, 3) + .split(';') + .map(|t| { + let (s, e) = t.split(':').nth(1).unwrap().split_once('-').unwrap(); + (s.parse().unwrap(), e.parse().unwrap()) + }) + .collect() + } + + #[test] + fn the_small_bed_gives_its_two_regions_then_the_hand_rows() { + let rows = small(SMALL_BED).unwrap(); + let ids: Vec<(&str, &str)> = rows.iter().map(|r| (cell(r, 0), cell(r, 2))).collect(); + assert_eq!( + ids[..2], + [("22q11.2_DGS_proximal_A-D", "chr22"), ("Xp22.31", "chrX")] + ); + let hand: Vec<&str> = ids[2..].iter().map(|x| x.0).collect(); + let want: Vec<&str> = HAND_ROWS.iter().map(|h| h.locus).collect(); + assert_eq!(hand, want, "the hand rows follow, in their order"); + assert!(rows.iter().all(|r| r.split('\t').count() == 22)); + assert!(rows.iter().all(|r| cell(r, 20) == "cnvpytor:cnvs")); + } + + #[test] + fn an_unknown_contig_stops_the_build() { + let e = small(&SMALL_BED.replace("chrx", "chrQ")).unwrap_err(); + assert!( + e.to_string().contains("'chrQ' is not a GRCh38 chromosome"), + "{e}" + ); + assert_eq!(e.code(), 2, "exit 2, as the Python builder did"); + } + + #[test] + fn no_target_piece_overlaps_a_breakpoint_row() { + let rows = small(SMALL_BED).unwrap(); + for (s, e) in spans(&rows[0]) { + assert!(!(s < 18_924_717 && 18_159_724 < e), "{s}-{e} on BP-A"); + assert!(!(s < 21_562_827 && 21_111_384 < e), "{s}-{e} on BP-D"); + } + // 2.19 Mb in pieces of at most 500 kb, the first labelled. + let t = cell(&rows[0], 3); + assert_eq!(t.split(';').count(), 5); + assert!(t.starts_with("22q11.2_DGS_proximal_A-D_t1:18924718-")); + assert!( + t.split(';') + .next() + .unwrap() + .ends_with(":22q11.2 recurrent (DGS) region (proximal, A-D) (includes TBX1)") + ); + } + + #[test] + fn controls_flank_the_cluster_and_never_sit_on_a_target() { + let rows = small(SMALL_BED).unwrap(); + // 22q11.2: the cluster runs from BP-A to BP-D, so the controls lie + // beyond those, 50 kb out. + assert_eq!( + cell(&rows[0], 4), + "22q11.2_DGS_proximal_A-D_L:18009724-18109724:flank left of the region;22q11.2_DGS_proximal_A-D_R:21612828-21712828:flank right of the region" + ); + for r in &rows { + if cell(r, 2) == "chrY" { + continue; + } + for c in cell(r, 4).split(';') { + let (s, e) = c.split(':').nth(1).unwrap().split_once('-').unwrap(); + let (cs, ce): (i64, i64) = (s.parse().unwrap(), e.parse().unwrap()); + for o in &rows { + if cell(o, 2) == cell(r, 2) { + assert!( + spans(o).iter().all(|&(ts, te)| !(ts < ce && cs < te)), + "{} control on {}", + cell(r, 0), + cell(o, 0) + ); + } + } + } + } + } + + #[test] + fn hand_rows_keep_their_spans_and_columns() { + let rows = small(SMALL_BED).unwrap(); + let get = |l: &str| rows.iter().find(|r| cell(r, 0) == l).unwrap().clone(); + assert_eq!(spans(&get("HBB_619bp_del")), [(5_225_388, 5_226_007)]); + assert_eq!( + spans(&get("HBD_HBB_Sicilian_del")), + [(5_219_877, 5_233_280)] + ); + let az = get("AZFc_DAZ"); + assert_eq!( + (cell(&az, 6), cell(&az, 7), cell(&az, 13), cell(&az, 14)), + ("sum", "pooled", "4", ".") + ); + assert_eq!(spans(&az).len(), 4, "the DAZ copies are never cut"); + assert!(cell(&az, 4).starts_with("AZFc_DAZ_c1:2935200-2982579:ZFY")); + let x = get("Xq28_MECP2"); + assert_eq!( + (cell(&x, 7), cell(&x, 13), cell(&x, 14)), + ("flanks", "8", "1.5") + ); + } + + #[test] + fn a_parascopy_model_is_cut_out_of_a_target() { + let mut cuts = vec![Cuts::default(); CHROMS.len()]; + let regions = read_clingen(SMALL_BED.lines(), &mut cuts).unwrap(); + cuts[21].parascopy.push((19_000_000, 19_100_000)); + let rows = Inputs { regions, cuts }.rows().unwrap(); + assert!( + spans(&rows[0]) + .iter() + .all(|&(s, e)| e <= 19_000_000 || s >= 19_100_000) + ); + assert!(spans(&rows[0]).iter().any(|&(_, e)| e == 19_000_000)); + } + + #[test] + fn a_parascopy_model_on_a_daz_copy_stops_the_build() { + let mut cuts = vec![Cuts::default(); CHROMS.len()]; + let regions = read_clingen(SMALL_BED.lines(), &mut cuts).unwrap(); + cuts[CHR_Y].parascopy.push((24_800_000, 24_800_100)); + let e = Inputs { regions, cuts }.rows().unwrap_err(); + assert!( + e.to_string() + .contains("AZFc_DAZ: DAZ3 overlaps a Parascopy model"), + "{e}" + ); + } + + #[test] + fn bad_rows_are_refused() { + let mut c = vec![Cuts::default(); CHROMS.len()]; + assert!( + read_clingen(["chr1\t1\t2"], &mut c) + .unwrap_err() + .to_string() + .contains("a row with 3 columns") + ); + let e = read_clingen(["chr1\t5\t5\tx\t0\t.\t5\t5\t255,111,0"], &mut c).unwrap_err(); + assert!(e.to_string().contains("x: empty span")); + let e = read_clingen(["chr1\t1\t5\tx\t0\t.\t1\t5\t1,2,3"], &mut c).unwrap_err(); + assert!(e.to_string().contains("itemRgb '1,2,3'")); + let e = read_clingen(["chr1\t1\t5\tx\t0\t.\t1\t5\t0,0,0"], &mut c).unwrap_err(); + assert!(e.to_string().contains("no region row")); + } + + #[test] + fn interval_helpers() { + assert_eq!( + merge(vec![(5, 8), (1, 3), (3, 4), (7, 10)]), + [(1, 4), (5, 10)] + ); + assert_eq!( + subtract(0, 20, &[(2, 4), (6, 8), (30, 40)]), + [(0, 2), (4, 6), (8, 20)] + ); + assert_eq!(subtract(3, 7, &[(0, 4), (6, 9)]), [(4, 6)]); + assert!(subtract(3, 7, &[(0, 9)]).is_empty()); + assert_eq!( + pieces(&[(0, 499), (0, 1_000_001)]), + [(0, 333_334), (333_334, 666_668), (666_668, 1_000_001)] + ); + assert_eq!(pieces(&[(0, 500)]), [(0, 500)]); + assert!(clear(4, 6, &[(0, 4), (6, 9)])); + assert!(!clear(3, 6, &[(0, 4), (6, 9)])); + assert_eq!(flank(0, 1_000_000, true, &[]), Some((850_000, 950_000))); + assert_eq!( + flank(0, 1_000_000, true, &[(900_000, 945_000)]), + Some((800_000, 900_000)) + ); + assert_eq!( + flank(0, 100_000, true, &[]), + None, + "past the chromosome start" + ); + } + + #[test] + fn locus_ids_and_labels() { + assert_eq!( + locus_id("22q11.2 recurrent (DGS) region (proximal, A-D) (includes TBX1)"), + "22q11.2_DGS_proximal_A-D" + ); + assert_eq!( + locus_id("Xp22.31 recurrent region (includes STS)"), + "Xp22.31" + ); + assert_eq!(locus_id("1q21.1 (a (b) c)"), "1q21.1_b"); + assert_eq!(label(" a;b\tc "), "a,b,c"); + } + + #[test] + fn a_repeated_locus_gets_a_number() { + let bed = "chr1\t10000000\t10100000\t1p (X) one\t0\t.\t0\t0\t255,111,0\nchr1\t20000000\t20100000\t1p (X) two\t0\t.\t0\t0\t255,111,0\n"; + let rows = small(bed).unwrap(); + assert_eq!((cell(&rows[0], 0), cell(&rows[1], 0)), ("1p_X", "1p_X_2")); + assert!(cell(&rows[1], 3).starts_with("1p_X_2_t1:")); + } + + #[test] + fn splice_keeps_the_head_and_check_counts_both_sides() { + let rows = vec!["a\t1".to_owned(), "b\t2".to_owned()]; + let old = b"# head\nx\n# --- interval rows: written by an older builder ---\nold\n"; + let new = splice(old, &rows); + assert_eq!( + new, + format!("# head\nx\n{MARKER}\na\t1\nb\t2\n").into_bytes() + ); + assert_eq!( + splice(b"x\n", &rows), + format!("x\n{MARKER}\na\t1\nb\t2\n").into_bytes() + ); + let added = [new.as_slice(), b"c\t3\n"].concat(); + assert_eq!(line_diff(&new, &added), (0, 1), "an added line is counted"); + assert_eq!(line_diff(&added, &new), (1, 0)); + let swapped = format!("# head\nx\n{MARKER}\nb\t2\na\t1\n").into_bytes(); + assert_eq!(line_diff(&new, &swapped), (0, 0)); + } +} diff --git a/rust/crates/og-dev/src/build/gtf.rs b/rust/crates/og-dev/src/build/gtf.rs new file mode 100644 index 000000000..89926e516 --- /dev/null +++ b/rust/crates/og-dev/src/build/gtf.rs @@ -0,0 +1,133 @@ +//! The GENCODE GTF reading the callability and SV rule builders share. +//! +//! A GTF line is read as bytes: its columns are found by their tabs, and an +//! attribute is found by a scan of the `key "value";` pairs of the ninth +//! column, only on the lines a builder keeps. GENCODE writes every pair as +//! `key "value"` or `key value` (a bare number), separated by `; `, and no +//! value holds a `;` or a `"`. + +use super::BuildError; + +/// One record of a GTF, by its columns. +pub(crate) struct Record<'a> { + pub seq: &'a [u8], + pub feature: &'a [u8], + start: &'a [u8], + end: &'a [u8], + pub attrs: &'a [u8], +} + +impl<'a> Record<'a> { + /// The record of a data line (without its line ending); `None` for a + /// comment. + pub fn parse(line: &'a [u8]) -> Result>, BuildError> { + if line.first() == Some(&b'#') { + return Ok(None); + } + let mut c = line.splitn(9, |&b| b == b'\t'); + let mut next = || { + c.next().ok_or_else(|| { + BuildError::Input(format!( + "GTF: a line with under 9 columns: {:?}", + String::from_utf8_lossy(line) + )) + }) + }; + let seq = next()?; + let _source = next()?; + let feature = next()?; + let start = next()?; + let end = next()?; + let (_score, _strand, _frame) = (next()?, next()?, next()?); + let attrs = next()?; + Ok(Some(Record { + seq, + feature, + start, + end, + attrs, + })) + } + + /// The 0-based start and the end. + pub fn span(&self) -> Result<(i64, i64), BuildError> { + Ok((number(self.start)? - 1, number(self.end)?)) + } + + /// The value of attribute `key`; the last one when it is given twice, + /// as a dict of the pairs keeps it. A value that is not UTF-8 is an + /// error. + pub fn attr(&self, key: &str) -> Result, BuildError> { + attr(self.attrs, key.as_bytes()) + .map(|v| { + std::str::from_utf8(v) + .map_err(|_| BuildError::Input(format!("GTF: {key} is not UTF-8: {v:?}"))) + }) + .transpose() + } +} + +/// A position column. +fn number(cell: &[u8]) -> Result { + std::str::from_utf8(cell) + .ok() + .and_then(|s| s.trim().parse().ok()) + .ok_or_else(|| { + BuildError::Input(format!( + "GTF: not a whole number: {:?}", + String::from_utf8_lossy(cell) + )) + }) +} + +/// The value of the last `key "value"` pair of an attribute column. +pub(crate) fn attr<'a>(attrs: &'a [u8], key: &[u8]) -> Option<&'a [u8]> { + let mut found = None; + for pair in attrs.split(|&b| b == b';') { + let pair = pair.trim_ascii_start(); + if let Some(rest) = pair.strip_prefix(key) + && let Some(quoted) = rest.strip_prefix(b" \"") + && let Some(end) = quoted.iter().position(|&b| b == b'"') + { + found = Some("ed[..end]); + } + } + found +} + +#[cfg(test)] +mod tests { + use super::*; + + const GENE: &[u8] = b"chr1\tHAVANA\tgene\t11\t20\t.\t+\t.\tgene_id \"ENSG1\"; gene_type \"protein_coding\"; gene_name \"A B\"; level 2; tag \"x\"; tag \"y\"; xgene_name \"no\";"; + + #[test] + fn a_gene_line_reads_its_columns_and_attributes() { + let r = Record::parse(GENE).unwrap().unwrap(); + assert_eq!((r.seq, r.feature), (&b"chr1"[..], &b"gene"[..])); + assert_eq!(r.span().unwrap(), (10, 20)); + assert_eq!(r.attr("gene_name").unwrap(), Some("A B"), "not xgene_name"); + assert_eq!( + r.attr("tag").unwrap(), + Some("y"), + "the last of a repeated key" + ); + assert_eq!( + r.attr("level").unwrap(), + None, + "an unquoted value is no pair" + ); + assert_eq!(r.attr("gene").unwrap(), None, "a key is matched whole"); + assert_eq!(r.attr("hgnc_id").unwrap(), None); + } + + #[test] + fn comments_pass_and_short_lines_stop() { + assert!(Record::parse(b"##description").unwrap().is_none()); + assert!(Record::parse(b"chr1\tx\tgene").is_err()); + let r = Record::parse(b"chr1\tx\tgene\tone\t2\t.\t+\t.\t") + .unwrap() + .unwrap(); + assert!(r.span().is_err()); + } +} diff --git a/rust/crates/og-dev/src/build/mod.rs b/rust/crates/og-dev/src/build/mod.rs new file mode 100644 index 000000000..b2098e3dc --- /dev/null +++ b/rust/crates/og-dev/src/build/mod.rs @@ -0,0 +1,298 @@ +//! `og dev build`: the builders of the asset tables OwnGenome commits. +//! +//! Each verb rebuilds one committed file from pinned public downloads that +//! a shell wrapper in `scripts/ci/` has already fetched and checked against +//! its sha256 (`scripts/ci/lib/fetch-pinned.sh`): `og` itself never +//! downloads. The committed file is the oracle, byte for byte. +//! +//! - `og dev build cn-intervals` ([`cn_intervals`]): the interval rows of +//! `bin/cn_loci.tsv` (`scripts/ci/build-cn-intervals.sh`). +//! - `og dev build callability` ([`callability`]): +//! `assets/callability_genes.tsv` (`scripts/ci/build-callability.sh`). +//! - `og dev build sv-rule-regions` ([`sv_rules`]): +//! `assets/sv_rule_regions.tsv` (`scripts/ci/build-sv-rule-regions.sh`). +//! +//! Exit 0 when the file was written (or `--check` found it right); 1 when +//! `--check` found it different, or when the callability or SV rule +//! downloads hold something their rules refuse (a CMRG gene with no GENCODE +//! gene, a symbol listed twice, not one MHC region), as the Python builders +//! exited; 2 for a usage error or an input that cannot be read, and for +//! every cn-intervals input the rules refuse, as its Python builder exited. + +pub mod callability; +pub mod cn_intervals; +mod gtf; +pub mod sv_rules; + +use og_io::{Lines, Utf8}; +use std::fmt; +use std::io::{self, BufRead, Write}; +use std::path::Path; + +const USAGE: &str = "usage: og dev build cn-intervals --clingen BED --segdups BED --gaps BED --satellites BED --parascopy TXT (--catalog FILE [--check] | --stdout)\n og dev build callability --work DIR --repo DIR --out FILE\n og dev build sv-rule-regions --work DIR --out FILE\n"; + +/// GRCh38's primary chromosomes in their sort order: chr1-22, X, Y. +pub(crate) const CHROMS: [&str; 24] = [ + "chr1", "chr2", "chr3", "chr4", "chr5", "chr6", "chr7", "chr8", "chr9", "chr10", "chr11", + "chr12", "chr13", "chr14", "chr15", "chr16", "chr17", "chr18", "chr19", "chr20", "chr21", + "chr22", "chrX", "chrY", +]; + +/// The index of a primary chromosome in [`CHROMS`], from its name's bytes. +pub(crate) fn chrom_index(name: impl AsRef<[u8]>) -> Option { + let rest = name.as_ref().strip_prefix(b"chr")?; + match rest { + b"X" => Some(22), + b"Y" => Some(23), + [d @ b'1'..=b'9'] => Some(usize::from(d - b'1')), + [a @ b'1'..=b'2', b @ b'0'..=b'9'] => { + let n = usize::from(a - b'0') * 10 + usize::from(b - b'0'); + (n <= 22).then(|| n - 1) + } + _ => None, + } +} + +/// Why a build stopped: exit 1 for downloads the rules refuse, exit 2 for +/// an input that cannot be read. +#[derive(Debug)] +pub enum BuildError { + /// The inputs were read, but they hold what the rules refuse. + Data(String), + /// An input could not be read or parsed. + Input(String), +} + +impl BuildError { + fn code(&self) -> u8 { + match self { + BuildError::Data(_) => 1, + BuildError::Input(_) => 2, + } + } +} + +impl fmt::Display for BuildError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + BuildError::Data(m) | BuildError::Input(m) => f.write_str(m), + } + } +} + +/// An input that cannot be read, named by its path. +pub(crate) fn input_error(path: &Path, e: &io::Error) -> BuildError { + BuildError::Input(crate::py_os_error(e, &path.to_string_lossy())) +} + +/// The lines of a plain or gzip file, without their endings, as Python's +/// text mode reads them. +pub(crate) fn lines(path: &Path, utf8: Utf8) -> Result>, BuildError> { + let r = og_io::open(path).map_err(|e| input_error(path, &e))?; + Ok(Lines::new(r, utf8)) +} + +/// Reads each line of `path` into `f`, stopping at the first error. +pub(crate) fn each_line( + path: &Path, + utf8: Utf8, + mut f: impl FnMut(&str) -> Result<(), BuildError>, +) -> Result<(), BuildError> { + for line in lines(path, utf8)? { + f(&line.map_err(|e| input_error(path, &e))?)?; + } + Ok(()) +} + +/// Reads each line of `path` (plain or gzip) into `f` as bytes, without +/// its `\n` or `\r\n`, stopping at the first error. For the big files, +/// where a text line per line would cost more than the reading. +pub(crate) fn each_raw_line( + path: &Path, + mut f: impl FnMut(&[u8]) -> Result<(), BuildError>, +) -> Result<(), BuildError> { + let mut r = og_io::open(path).map_err(|e| input_error(path, &e))?; + let mut buf = Vec::with_capacity(1 << 12); + loop { + buf.clear(); + if r.read_until(b'\n', &mut buf) + .map_err(|e| input_error(path, &e))? + == 0 + { + return Ok(()); + } + let mut line = buf.as_slice(); + if let Some(l) = line.strip_suffix(b"\n") { + line = l.strip_suffix(b"\r").unwrap_or(l); + } + f(line)?; + } +} + +/// A whole number of a text column, as Python's `int()` reads one. +pub(crate) fn int(cell: &str, what: &str) -> Result { + pycompat::text::strip(cell, None) + .parse() + .map_err(|_| BuildError::Input(format!("{what}: not a whole number: {cell:?}"))) +} + +/// Takes `--flag VALUE` pairs and bare `--switch`es from `args`. +pub(crate) struct Args { + values: Vec<(String, String)>, + switches: Vec, +} + +impl Args { + pub(crate) fn parse( + args: &[String], + flags: &[&str], + switches: &[&str], + ) -> Result { + let mut out = Args { + values: Vec::new(), + switches: Vec::new(), + }; + let mut it = args.iter(); + while let Some(a) = it.next() { + if flags.contains(&a.as_str()) { + let v = it + .next() + .ok_or_else(|| format!("argument {a}: expected one argument"))?; + if out.values.iter().any(|(k, _)| k == a) { + return Err(format!("argument {a}: given twice")); + } + out.values.push((a.clone(), v.clone())); + } else if switches.contains(&a.as_str()) { + out.switches.push(a.clone()); + } else { + return Err(format!("unrecognized arguments: {a}")); + } + } + Ok(out) + } + + pub(crate) fn get(&self, flag: &str) -> Option<&str> { + self.values + .iter() + .find(|(k, _)| k == flag) + .map(|(_, v)| v.as_str()) + } + + pub(crate) fn need(&self, flag: &str) -> Result<&str, String> { + self.get(flag) + .ok_or_else(|| format!("the following arguments are required: {flag}")) + } + + pub(crate) fn has(&self, switch: &str) -> bool { + self.switches.iter().any(|s| s == switch) + } +} + +/// `og dev build VERB ARGS`; returns the exit code. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let result = match args.first().map(String::as_str) { + Some("cn-intervals") => cn_intervals::cli(&args[1..], out, err), + Some("callability") => callability::cli(&args[1..], err), + Some("sv-rule-regions") => sv_rules::cli(&args[1..], err), + Some("-h" | "--help") => { + let _ = out.write_all(USAGE.as_bytes()); + return 0; + } + Some(other) => Err(Failure::Usage(format!("unknown builder '{other}'"))), + None => Err(Failure::Usage( + "the following arguments are required: builder".to_owned(), + )), + }; + match result { + Ok(code) => code, + Err(Failure::Usage(m)) => { + let _ = writeln!(err, "{USAGE}og dev build: error: {m}"); + 2 + } + Err(Failure::Build(e)) => { + let _ = writeln!(err, "ERROR: {e}"); + e.code() + } + } +} + +/// How a builder's command line ended badly. +pub(crate) enum Failure { + Usage(String), + Build(BuildError), +} + +impl From for Failure { + fn from(e: BuildError) -> Self { + Failure::Build(e) + } +} + +impl From for Failure { + fn from(m: String) -> Self { + Failure::Usage(m) + } +} + +/// Writes `bytes` to `path` through a temporary file renamed over it. +pub(crate) fn write_atomic(path: &Path, bytes: &[u8]) -> Result<(), BuildError> { + let mut f = og_io::AtomicFile::create(path).map_err(|e| input_error(path, &e))?; + f.write_all(bytes).map_err(|e| input_error(path, &e))?; + f.commit().map_err(|e| input_error(path, &e)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn s(v: &[&str]) -> Vec { + v.iter().map(|x| (*x).to_owned()).collect() + } + + #[test] + fn unknown_builder_is_a_usage_error() { + let (mut o, mut e) = (Vec::new(), Vec::new()); + assert_eq!(cli(&s(&["nope"]), &mut o, &mut e), 2); + assert!( + String::from_utf8(e) + .unwrap() + .contains("unknown builder 'nope'") + ); + assert_eq!(cli(&[], &mut o, &mut Vec::new()), 2); + } + + #[test] + fn args_refuse_a_repeated_or_unknown_flag() { + assert!(Args::parse(&s(&["--out", "a", "--out", "b"]), &["--out"], &[]).is_err()); + assert!(Args::parse(&s(&["--what"]), &["--out"], &[]).is_err()); + assert!(Args::parse(&s(&["--out"]), &["--out"], &[]).is_err()); + let a = Args::parse(&s(&["--out", "a", "--check"]), &["--out"], &["--check"]).unwrap(); + assert_eq!(a.get("--out"), Some("a")); + assert!(a.has("--check")); + } + + #[test] + fn chromosome_names() { + for (i, c) in CHROMS.iter().enumerate() { + assert_eq!(chrom_index(c), Some(i)); + } + for c in [ + "chr0", + "chr23", + "chr01", + "chrM", + "chr1_KI270706v1_random", + "1", + "chrx", + "", + ] { + assert_eq!(chrom_index(c), None, "{c}"); + } + } + + #[test] + fn int_reads_as_python_does() { + assert_eq!(int(" 12 ", "x").unwrap(), 12); + assert!(int("1.5", "x").is_err()); + } +} diff --git a/rust/crates/og-dev/src/build/sv_rules.rs b/rust/crates/og-dev/src/build/sv_rules.rs new file mode 100644 index 000000000..7004f02b7 --- /dev/null +++ b/rust/crates/og-dev/src/build/sv_rules.rs @@ -0,0 +1,649 @@ +//! `og dev build sv-rule-regions`: `assets/sv_rule_regions.tsv`, the GRCh38 +//! regions where `og sv postfilter` overrides AnnotSV's class 4 or 5. +//! +//! All on chr1-22, X and Y: +//! +//! - `tandem_repeat`: every TRExplorer v2.1 repeat of 50 bp or more (the +//! shortest SV) that overlaps a coding exon of a MANE Select transcript; +//! name `GENE:MOTIF` (a motif over 12 bases as its length); +//! - `ig_tr_locus`: the span of GENCODE's IG and TR genes of each of IGH, +//! IGK, IGL, TRA (with TRD inside it), TRB and TRG: per locus the largest +//! group of genes on one chromosome with gaps under 1 Mb, so the orphon +//! copies on other chromosomes stay out; +//! - `mhc`: the MHC region of NCBI's assembly regions report; +//! - `gene_alias`: the GENCODE span of each HGNC pseudogene whose alias or +//! previous symbol is the approved symbol of a gene with a GenCC +//! curation; name `ALIAS|PSEUDOGENE|its HGNC id|the HGNC id of the gene +//! that holds ALIAS as its symbol`. +//! +//! A gene's span is that of the first GENCODE record of its HGNC id, and the +//! `_PAR_Y` copies of the pseudoautosomal genes are skipped, as the +//! callability builder does: the Python this replaces kept the last record, +//! so a PAR pseudogene would have taken its chrY span (none does in the +//! pinned files, so the committed table is the same). +//! +//! The GTF and the catalog are streamed once. The MANE Select coding exons +//! of each chromosome are sorted by start with a running maximum of their +//! ends, so each repeat finds the exons it overlaps by binary search. + +use super::gtf::Record; +use super::{Args, BuildError, CHROMS, Failure, chrom_index, each_line, each_raw_line, int}; +use og_io::{Lines, Utf8}; +use pycompat::csv::{DictReader, Quoting}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::io::{self, Write}; +use std::path::Path; + +/// The tag of a MANE Select transcript's records. +const MANE: &[u8] = b"tag \"MANE_Select\""; +const MIN_TR: i64 = 50; +const LOCUS_GAP: i64 = 1_000_000; +const SRC_TR: &str = "TRExplorer v2.1; GENCODE 50 MANE Select CDS"; +const SRC_IG: &str = "GENCODE 50 IG and TR genes"; +const SRC_MHC: &str = "GRCh38 assembly regions (GCA_000001405.15)"; +const SRC_ALIAS: &str = "HGNC 2026-10-06; GENCODE 50"; + +/// One row of the table. +struct Row { + chrom: usize, + start: i64, + end: i64, + rule: &'static str, + action: &'static str, + name: String, + source: &'static str, +} + +/// What the GTF gives. +#[derive(Default)] +struct Gtf { + /// HGNC id -> (chrom, start, end) of its first record. + span: HashMap, + /// IG or TR locus -> its genes as (chrom name, start, end). + ig: BTreeMap>, + /// Per chromosome, the MANE Select coding exons: (start, end, gene). + cds: Vec>, +} + +/// The IG or TR locus of a gene, from its type and symbol. +fn ig_locus(gene_type: &str, name: &str) -> Option { + let ig_type = gene_type.starts_with("IG_") || gene_type.starts_with("TR_"); + let b = name.as_bytes(); + let ig_name = b.len() >= 3 + && matches!( + (b[0], b[1], b[2]), + (b'I', b'G', b'H' | b'K' | b'L') | (b'T', b'R', b'A' | b'B' | b'D' | b'G') + ); + (ig_type && ig_name).then(|| { + if name.starts_with("TRD") { + "TRA".to_owned() + } else { + name[..3].to_owned() + } + }) +} + +fn read_gtf(path: &Path) -> Result { + let mut g = Gtf { + cds: vec![Vec::new(); CHROMS.len()], + ..Gtf::default() + }; + each_raw_line(path, |line| { + let Some(r) = Record::parse(line)? else { + return Ok(()); + }; + let Some(chrom) = chrom_index(r.seq) else { + return Ok(()); + }; + match r.feature { + b"gene" => { + let (start, end) = r.span()?; + if let Some(h) = r.attr("hgnc_id")?.filter(|h| !h.is_empty()) + && !r.attr("gene_id")?.is_some_and(|id| id.ends_with("_PAR_Y")) + { + g.span.entry(h.to_owned()).or_insert((chrom, start, end)); + } + let (name, ty) = ( + r.attr("gene_name")?.unwrap_or(""), + r.attr("gene_type")?.unwrap_or(""), + ); + if let Some(locus) = ig_locus(ty, name) { + g.ig.entry(locus) + .or_default() + .push((CHROMS[chrom], start, end)); + } + } + b"CDS" if r.attrs.windows(MANE.len()).any(|w| w == MANE) => { + let name = r.attr("gene_name")?.ok_or_else(|| { + BuildError::Input(format!( + "GTF: a MANE Select CDS with no gene_name: {:?}", + String::from_utf8_lossy(line) + )) + })?; + let (start, end) = r.span()?; + g.cds[chrom].push((start, end, name.to_owned())); + } + _ => {} + } + Ok(()) + })?; + Ok(g) +} + +/// The MANE Select coding exons of one chromosome, sorted, with the +/// running maximum of their ends. +struct Exons { + iv: Vec<(i64, i64, String)>, + max_end: Vec, +} + +impl Exons { + fn new(mut iv: Vec<(i64, i64, String)>) -> Exons { + iv.sort_unstable(); + let mut m = i64::MIN; + let max_end = iv + .iter() + .map(|x| { + m = m.max(x.1); + m + }) + .collect(); + Exons { iv, max_end } + } + + /// The genes of the exons that overlap `[s, e)`, sorted. + fn genes(&self, s: i64, e: i64) -> BTreeSet<&str> { + let mut out = BTreeSet::new(); + let mut j = self.iv.partition_point(|x| x.0 < e); + while j > 0 && self.max_end[j - 1] > s { + j -= 1; + if self.iv[j].1 > s { + out.insert(self.iv[j].2.as_str()); + } + } + out + } +} + +/// tandem_repeat rows: catalog repeats of `MIN_TR` bp or more on a MANE +/// Select coding exon. +fn tandem_repeats( + path: &Path, + cds: Vec>, + rows: &mut Vec, +) -> Result<(), BuildError> { + let index: Vec> = cds + .into_iter() + .map(|iv| (!iv.is_empty()).then(|| Exons::new(iv))) + .collect(); + each_raw_line(path, |line| { + let mut c = line.splitn(5, |&b| b == b'\t'); + let Some(ex) = c + .next() + .and_then(chrom_index) + .and_then(|i| index[i].as_ref().map(|x| (i, x))) + else { + return Ok(()); + }; + let short = || { + BuildError::Input(format!( + "{}: a row with under 4 columns: {:?}", + path.display(), + String::from_utf8_lossy(line) + )) + }; + let (s, e) = (c.next().ok_or_else(short)?, c.next().ok_or_else(short)?); + let (s, e) = (bytes_int(s)?, bytes_int(e)?); + if e - s < MIN_TR { + return Ok(()); + } + let genes = ex.1.genes(s, e); + if !genes.is_empty() { + let motif = std::str::from_utf8(c.next().ok_or_else(short)?).map_err(|_| { + BuildError::Input(format!("{}: a motif that is not UTF-8", path.display())) + })?; + let n = motif.chars().count(); + let motif = if n <= 12 { + motif.to_owned() + } else { + format!("{n}bp motif") + }; + rows.push(Row { + chrom: ex.0, + start: s, + end: e, + rule: "tandem_repeat", + action: "class_3", + name: format!( + "{}:{motif}", + genes.into_iter().collect::>().join("/") + ), + source: SRC_TR, + }); + } + Ok(()) + }) +} + +/// A position of the catalog. +fn bytes_int(cell: &[u8]) -> Result { + int(&String::from_utf8_lossy(cell), "catalog position") +} + +/// ig_tr_locus rows: per locus the largest group of genes on one +/// chromosome with gaps under `LOCUS_GAP` (the first such group on a tie). +fn ig_loci(ig: BTreeMap>, rows: &mut Vec) { + for (locus, mut genes) in ig { + genes.sort_unstable(); + // (chrom, start, end, genes) + let mut groups: Vec<(&str, i64, i64, usize)> = Vec::new(); + for (chrom, s, e) in genes { + match groups.last_mut() { + Some(g) if g.0 == chrom && s - g.2 < LOCUS_GAP => { + g.2 = g.2.max(e); + g.3 += 1; + } + _ => groups.push((chrom, s, e, 1)), + } + } + let mut best = groups[0]; + for g in &groups[1..] { + if g.3 > best.3 { + best = *g; + } + } + rows.push(Row { + chrom: chrom_index(best.0).expect("only chr1-22, X and Y are read"), + start: best.1, + end: best.2, + rule: "ig_tr_locus", + action: "class_3", + name: locus, + source: SRC_IG, + }); + } +} + +/// The mhc row: the assembly report's one MHC region (1-based, inclusive). +fn mhc(path: &Path) -> Result { + let mut found: BTreeSet> = BTreeSet::new(); + each_line(path, Utf8::Strict, |line| { + if line.starts_with("MHC\t") { + found.insert(line.split('\t').take(4).map(str::to_owned).collect()); + } + Ok(()) + })?; + let one = match found.len() { + 1 => found.pop_first().unwrap_or_default(), + n => { + return Err(BuildError::Data(format!( + "want one MHC region in the assembly report, got {n}: {found:?}" + ))); + } + }; + let bad = || { + BuildError::Input(format!( + "{}: an MHC row with under 4 columns: {one:?}", + path.display() + )) + }; + let (chrom, s, e) = ( + one.get(1).ok_or_else(bad)?, + one.get(2).ok_or_else(bad)?, + one.get(3).ok_or_else(bad)?, + ); + let name = format!("chr{chrom}"); + Ok(Row { + chrom: chrom_index(&name).ok_or_else(|| { + BuildError::Input(format!("{}: the MHC is on {name}", path.display())) + })?, + start: int(s, "MHC start")? - 1, + end: int(e, "MHC end")?, + rule: "mhc", + action: "class_3", + name: "MHC".to_owned(), + source: SRC_MHC, + }) +} + +/// The lines of a file with their endings, as a file opened with +/// `newline=""` hands them to `csv`. +struct RawLines(Lines>); + +impl Iterator for RawLines { + type Item = io::Result; + fn next(&mut self) -> Option { + let mut s = String::new(); + match self.0.read_line(&mut s) { + Ok(true) => Some(Ok(s)), + Ok(false) => None, + Err(e) => Some(Err(e)), + } + } +} + +/// gene_alias rows: a pseudogene that carries a disease gene's symbol as an +/// alias or previous symbol. +fn gene_aliases( + path: &Path, + span: &HashMap, + rows: &mut Vec, +) -> Result<(), BuildError> { + let csv_error = |e: pycompat::csv::Error| BuildError::Input(format!("{}: {e}", path.display())); + // symbol -> (HGNC id, has a GenCC curation); the last row of a symbol. + let mut approved: HashMap = HashMap::new(); + // (HGNC id, symbol, the alias and previous symbols) of the pseudogenes + // GENCODE places. + let mut pseudo: Vec<(String, String, BTreeSet)> = Vec::new(); + let lines = RawLines(super::lines(path, Utf8::Strict)?); + for row in DictReader::new(lines, '\t', Quoting::Minimal) { + let row = row.map_err(csv_error)?; + let get = |k: &str| { + row.get(k).flatten().ok_or_else(|| { + BuildError::Input(format!("{}: a row with no {k} cell", path.display())) + }) + }; + let (symbol, hgnc) = (get("symbol")?, get("hgnc_id")?); + approved.insert( + symbol.to_owned(), + (hgnc.to_owned(), !get("gencc")?.is_empty()), + ); + if get("locus_group")? == "pseudogene" && span.contains_key(hgnc) { + let names = format!("{}|{}", get("alias_symbol")?, get("prev_symbol")?); + let syms = names + .split('|') + .map(|x| x.trim_matches('"')) + .filter(|x| !x.is_empty()) + .map(str::to_owned) + .collect(); + pseudo.push((hgnc.to_owned(), symbol.to_owned(), syms)); + } + } + for (hgnc, symbol, syms) in pseudo { + for sym in syms { + if let Some((other, true)) = approved.get(&sym) + && *other != hgnc + { + let (chrom, start, end) = span[&hgnc]; + rows.push(Row { + chrom, + start, + end, + rule: "gene_alias", + action: "remap_gene", + name: format!("{sym}|{symbol}|{hgnc}|{other}"), + source: SRC_ALIAS, + }); + } + } + } + Ok(()) +} + +/// The table's text and its row counts line, from the downloads in `work`. +pub fn build(work: &Path) -> Result<(String, String), BuildError> { + let gtf = read_gtf(&work.join("gencode.gtf.gz"))?; + let mut rows = Vec::new(); + tandem_repeats(&work.join("trx.bed.gz"), gtf.cds, &mut rows)?; + ig_loci(gtf.ig, &mut rows); + rows.push(mhc(&work.join("regions.txt"))?); + gene_aliases(&work.join("hgnc.txt"), >f.span, &mut rows)?; + rows.sort_by(|a, b| { + (a.chrom, a.start, a.end, a.rule, &a.name).cmp(&(b.chrom, b.start, b.end, b.rule, &b.name)) + }); + let mut counts: BTreeMap<&str, usize> = BTreeMap::new(); + for r in &rows { + *counts.entry(r.rule).or_default() += 1; + } + let counts: Vec = counts.iter().map(|(k, n)| format!("{k} {n}")).collect(); + let summary = format!("Rows: {}", counts.join(", ")); + let mut text = String::from( + "# assets/sv_rule_regions.tsv: the GRCh38 regions where og sv postfilter overrides AnnotSV's class 4\n\ + # or 5. Written by scripts/ci/build-sv-rule-regions.sh, which says how each rule's rows are chosen;\n\ + # rebuild it there, do not edit it by hand. start is 0-based and end 1-based, as in a BED file.\n", + ); + text.push_str(&format!( + "# {summary}.\n#chrom\tstart\tend\trule\taction\tname\tsource\n" + )); + for r in &rows { + text.push_str(&format!( + "{}\t{}\t{}\t{}\t{}\t{}\t{}\n", + CHROMS[r.chrom], r.start, r.end, r.rule, r.action, r.name, r.source + )); + } + Ok((text, summary)) +} + +pub(super) fn cli(args: &[String], err: &mut dyn Write) -> Result { + let a = Args::parse(args, &["--work", "--out"], &[])?; + let (work, out) = (a.need("--work")?, a.need("--out")?); + let (text, summary) = build(Path::new(work))?; + super::write_atomic(Path::new(out), text.as_bytes())?; + let _ = writeln!(err, "{summary}"); + Ok(0) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + fn gz(text: &str) -> Vec { + let mut g = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + g.write_all(text.as_bytes()).unwrap(); + g.finish().unwrap() + } + + fn gene(seq: &str, s: u32, e: u32, id: &str, ty: &str, name: &str, h: &str) -> String { + format!( + "{seq}\tX\tgene\t{s}\t{e}\t.\t+\t.\tgene_id \"{id}\"; gene_type \"{ty}\"; gene_name \"{name}\"; hgnc_id \"{h}\";\n" + ) + } + + fn cds(seq: &str, s: u32, e: u32, name: &str, mane: bool) -> String { + let tag = if mane { " tag \"MANE_Select\";" } else { "" }; + format!("{seq}\tX\tCDS\t{s}\t{e}\t.\t+\t0\tgene_id \"x\"; gene_name \"{name}\";{tag}\n") + } + + /// A work folder with a few rows of each rule; `par_first` puts the + /// _PAR_Y copy of the pseudogene PSP before its chrX record. + fn fixture(dir: &Path, par_first: bool) { + fs::create_dir_all(dir).unwrap(); + let px = gene( + "chrX", + 1001, + 2000, + "GP", + "unprocessed_pseudogene", + "PSP", + "HGNC:30", + ); + let py = gene( + "chrY", + 1001, + 2000, + "GP_PAR_Y", + "unprocessed_pseudogene", + "PSP", + "HGNC:30", + ); + let par = if par_first { + [py.clone(), px.clone()] + } else { + [px, py] + }; + let gtf = [ + "##description: test\n".to_owned(), + gene("chr1", 1001, 3000, "G1", "protein_coding", "AAA", "HGNC:1"), + cds("chr1", 1101, 1300, "AAA", true), + cds("chr1", 1251, 1400, "AAB", true), + cds("chr1", 2001, 2400, "AAA", false), + gene( + "chr14", + 100_001, + 100_500, + "I1", + "IG_V_gene", + "IGHV1", + "HGNC:10", + ), + gene( + "chr14", + 900_001, + 900_500, + "I2", + "IG_C_gene", + "IGHM", + "HGNC:11", + ), + gene( + "chr15", + 1, + 500, + "I3", + "IG_V_pseudogene", + "IGHV1OR15", + "HGNC:12", + ), + gene( + "chr14", + 20_001, + 20_500, + "T1", + "TR_V_gene", + "TRDV1", + "HGNC:13", + ), + gene( + "chr14", + 10_001, + 10_500, + "T2", + "TR_V_gene", + "TRAV1", + "HGNC:14", + ), + gene( + "chr2", + 5001, + 6000, + "G2", + "processed_pseudogene", + "PSA", + "HGNC:20", + ), + par[0].clone(), + par[1].clone(), + ] + .concat(); + fs::write(dir.join("gencode.gtf.gz"), gz(>f)).unwrap(); + fs::write( + dir.join("trx.bed.gz"), + gz("chr1\t1000\t1049\tCAG\nchr1\t1200\t1260\tCAG\nchr1\t1280\t1340\tAAAAAAAAAAAAAT\nchr1\t2100\t2200\tGC\nchrM\t1\t100\tA\n"), + ) + .unwrap(); + fs::write( + dir.join("regions.txt"), + "# Region-Name\tChromosome\tStart\tStop\tRole\nMHC\t6\t28510120\t33480577\tx\r\nMHC\t6\t28510120\t33480577\tx\r\nOTHER\t1\t1\t2\tx\n", + ) + .unwrap(); + fs::write( + dir.join("hgnc.txt"), + "hgnc_id\tsymbol\tlocus_group\talias_symbol\tprev_symbol\tgencc\n\ + HGNC:1\tAAA\tprotein-coding gene\t\t\t\"curated\"\n\ + HGNC:2\tBBB\tprotein-coding gene\t\t\t\n\ + HGNC:3\tCCC\tprotein-coding gene\t\t\tyes\n\ + HGNC:20\tPSA\tpseudogene\t\"AAA|BBB\"\tPSA\t\n\ + HGNC:30\tPSP\tpseudogene\tCCC\t\t\n\ + HGNC:40\tPSZ\tpseudogene\tAAA\t\t\n", + ) + .unwrap(); + } + + #[test] + fn each_rule_gives_its_rows() { + let dir = std::env::temp_dir().join(format!("og-svrules-{}", std::process::id())); + fixture(&dir, false); + let (text, summary) = build(&dir).unwrap(); + fs::remove_dir_all(&dir).unwrap(); + let body: Vec<&str> = text.lines().skip(4).collect(); + assert_eq!( + body, + [ + "#chrom\tstart\tend\trule\taction\tname\tsource", + "chr1\t1200\t1260\ttandem_repeat\tclass_3\tAAA/AAB:CAG\tTRExplorer v2.1; GENCODE 50 MANE Select CDS", + "chr1\t1280\t1340\ttandem_repeat\tclass_3\tAAA/AAB:14bp motif\tTRExplorer v2.1; GENCODE 50 MANE Select CDS", + "chr2\t5000\t6000\tgene_alias\tremap_gene\tAAA|PSA|HGNC:20|HGNC:1\tHGNC 2026-10-06; GENCODE 50", + "chr6\t28510119\t33480577\tmhc\tclass_3\tMHC\tGRCh38 assembly regions (GCA_000001405.15)", + "chr14\t10000\t20500\tig_tr_locus\tclass_3\tTRA\tGENCODE 50 IG and TR genes", + "chr14\t100000\t900500\tig_tr_locus\tclass_3\tIGH\tGENCODE 50 IG and TR genes", + "chrX\t1000\t2000\tgene_alias\tremap_gene\tCCC|PSP|HGNC:30|HGNC:3\tHGNC 2026-10-06; GENCODE 50", + ] + ); + assert_eq!( + summary, + "Rows: gene_alias 2, ig_tr_locus 2, mhc 1, tandem_repeat 2" + ); + assert!(text.contains("\n# Rows: gene_alias 2, ig_tr_locus 2, mhc 1, tandem_repeat 2.\n")); + } + + #[test] + fn a_par_pseudogene_keeps_its_chrx_span_whichever_record_comes_first() { + let dir = std::env::temp_dir().join(format!("og-svrules-par-{}", std::process::id())); + fixture(&dir, true); + let (text, _) = build(&dir).unwrap(); + fs::remove_dir_all(&dir).unwrap(); + assert!( + text.contains("\nchrX\t1000\t2000\tgene_alias\tremap_gene\tCCC|PSP|HGNC:30|HGNC:3\t"), + "{text}" + ); + assert!(!text.contains("\nchrY\t"), "{text}"); + } + + #[test] + fn two_mhc_regions_stop_the_build() { + let dir = std::env::temp_dir().join(format!("og-svrules-mhc-{}", std::process::id())); + fixture(&dir, false); + fs::write( + dir.join("regions.txt"), + "MHC\t6\t1\t2\tx\nMHC\t6\t1\t3\tx\n", + ) + .unwrap(); + let e = build(&dir).err().unwrap(); + fs::remove_dir_all(&dir).unwrap(); + assert!(matches!(e, BuildError::Data(_))); + assert!( + e.to_string() + .starts_with("want one MHC region in the assembly report, got 2"), + "{e}" + ); + } + + #[test] + fn exon_lookup_finds_every_overlap() { + let ex = Exons::new(vec![ + (0, 100, "A".into()), + (10, 20, "B".into()), + (50, 60, "C".into()), + (200, 300, "D".into()), + ]); + assert_eq!( + ex.genes(15, 55).into_iter().collect::>(), + ["A", "B", "C"] + ); + assert_eq!(ex.genes(100, 200).len(), 0, "half-open on both sides"); + assert_eq!( + ex.genes(99, 201).into_iter().collect::>(), + ["A", "D"] + ); + } + + #[test] + fn ig_loci_by_type_and_symbol() { + assert_eq!(ig_locus("IG_V_gene", "IGKV1").as_deref(), Some("IGK")); + assert_eq!(ig_locus("TR_D_gene", "TRDD1").as_deref(), Some("TRA")); + assert_eq!(ig_locus("protein_coding", "IGHM"), None); + assert_eq!(ig_locus("TR_V_gene", "TRX1"), None); + assert_eq!(ig_locus("IG_V_gene", "IG"), None); + } +} diff --git a/rust/crates/og-dev/src/lib.rs b/rust/crates/og-dev/src/lib.rs index 55c93fdf0..88c683e45 100644 --- a/rust/crates/og-dev/src/lib.rs +++ b/rust/crates/og-dev/src/lib.rs @@ -12,7 +12,10 @@ //! the report carries the report-only error strategy. //! - `og dev remnants` ([`remnants`]): does each Python file kept only for //! its importers (`tests/parity/remnants.tsv`) hold nothing else? +//! - `og dev build cn-intervals|callability|sv-rule-regions` ([`build`]): +//! rebuild a committed asset table from its pinned downloads. +pub mod build; pub mod contract; pub mod parity; pub mod remnants; @@ -22,7 +25,7 @@ mod zip; use std::io::Write; -const USAGE: &str = "usage: og dev parity-compare KIND ORACLE NEW\n og dev parity-compare --self-test\n og dev contract --static [--repo DIR] [--contract FILE]\n og dev contract --outputs DIR --leg bash|nextflow --sample S [--observe]\n og dev schema SCHEMA.json DOC.json\n og dev report-only [--repo DIR]\n og dev remnants [--list FILE] [--root DIR]\n"; +const USAGE: &str = "usage: og dev parity-compare KIND ORACLE NEW\n og dev parity-compare --self-test\n og dev contract --static [--repo DIR] [--contract FILE]\n og dev contract --outputs DIR --leg bash|nextflow --sample S [--observe]\n og dev schema SCHEMA.json DOC.json\n og dev report-only [--repo DIR]\n og dev remnants [--list FILE] [--root DIR]\n og dev build cn-intervals|callability|sv-rule-regions [args]\n"; /// Python's `str(OSError)`: `[Errno 2] No such file or directory: 'path'`. pub(crate) fn py_os_error(e: &std::io::Error, path: &str) -> String { @@ -57,6 +60,7 @@ pub fn main(args: &[String]) -> u8 { Some("contract") => run(contract::cli, &args[1..]), Some("schema") => run(schema::cli, &args[1..]), Some("report-only") => run(report_only::cli, &args[1..]), + Some("build") => run(build::cli, &args[1..]), Some("-h" | "--help") => { print!("{USAGE}"); 0 diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs index 234ea7cac..a34aa302f 100644 --- a/rust/og/src/main.rs +++ b/rust/og/src/main.rs @@ -74,6 +74,7 @@ const USAGE: &str = "usage: og [args] og exomiser report [args] og lookup query|resolve|report [args] og dev parity-compare|contract|schema|report-only|remnants [args] + og dev build cn-intervals|callability|sv-rule-regions [args] og dev freshness [--out FILE] [--sections a,b] | --update-issue BODY --repo O/N | --self-test og version og --tree-hash From f47eb8a998025a300368b015e69ca5d59910bd7f Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:27:46 +0100 Subject: [PATCH 02/17] build(ci): switch the asset builders to og dev build, one fetch rule build-callability.sh and build-sv-rule-regions.sh keep their downloads and hand the folder to og dev build; their Python heredocs are gone. build-cn-intervals.py becomes build-cn-intervals.sh with the same --list, --cache and --check. The three share scripts/ci/lib/fetch-pinned.sh: a cached file with the right sha256 is read in place, a 403 or 404 fails at once, anything else is retried for about five minutes. The marker line of bin/cn_loci.tsv names the new script; no row changes. The cn-intervals case, the e2e data fill and the converter checks of tests/test_copy_number.py call the new script and og. Bead: pgp-bs9u.35 --- CHANGELOG.md | 6 + bin/cn_loci.tsv | 2 +- bin/copy_number.py | 2 +- docs/cn-loci.md | 6 +- docs/rust.md | 2 +- docs/testing.md | 2 +- scripts/ci/build-callability.sh | 218 +------------- scripts/ci/build-cn-intervals.py | 423 ---------------------------- scripts/ci/build-cn-intervals.sh | 97 +++++++ scripts/ci/build-sv-rule-regions.sh | 171 ++--------- scripts/ci/e2e-case-paths.tsv | 6 +- scripts/ci/e2e-run.sh | 2 +- scripts/ci/lib/fetch-pinned.sh | 61 ++++ tests/e2e/cn-intervals-1-catalog.sh | 10 +- tests/test_copy_number.py | 77 ++--- tests/test_e2e_data_cache.sh | 2 +- versions.env | 2 +- 17 files changed, 255 insertions(+), 834 deletions(-) delete mode 100755 scripts/ci/build-cn-intervals.py create mode 100755 scripts/ci/build-cn-intervals.sh create mode 100644 scripts/ci/lib/fetch-pinned.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e21d1ac8..7ce7f1507 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The three builders of the committed asset tables now run in the Rust `og` binary: `og dev build cn-intervals` (the interval rows of `bin/cn_loci.tsv`), `og dev build callability` (`assets/callability_genes.tsv`) and `og dev build sv-rule-regions` (`assets/sv_rule_regions.tsv`). They replace `scripts/ci/build-cn-intervals.py`, which is gone (the script is now `scripts/ci/build-cn-intervals.sh`), and the Python inside `scripts/ci/build-callability.sh` and `scripts/ci/build-sv-rule-regions.sh`. The scripts still download the pinned files, and the three tables are the same, byte for byte, as the committed ones. With the downloads in the cache, the callability table takes 2.5 s and 27 MB instead of 5.5 s and 39 MB, and the SV rule table 4.9 s and 30 MB instead of 10.5 s and 217 MB. What changes on purpose: + - The three scripts share one download function, `scripts/ci/lib/fetch-pinned.sh`. A file the cache folder already holds with the right sha256 is read where it is, not copied. A 403 or 404 fails at once; any other failure is tried for about five minutes, 30 s apart. `build-cn-intervals.py` tried three times, 30 s apart. + - The marker line of `bin/cn_loci.tsv` names `scripts/ci/build-cn-intervals.sh`. No row changes. + - `build-cn-intervals.sh --check` counts the lines only in the file and the lines only in the rebuild. The Python counted only the lines the rebuild lacks, so a catalog with an added line said "0 line(s) differ". + - The SV rule builder takes each gene's span from the first GENCODE record of its HGNC id and skips the `_PAR_Y` copies, as the callability builder does. The Python kept the last record, so a pseudogene in a pseudoautosomal region would have taken its chrY span. No gene of the pinned files is one, so the table does not change. + - The Rust `og` binary has a first part of the summary collector: `og summary collect --only a`. It reads seven sections of summary.json (coverage, callability, sex_check, sample_qc, mosaic_sex_loss, read_count and variants), the run status, the manifest and each section's state. Nothing calls it yet: the reports still read the summary that `bin/collect_summary.py` writes. The sections are the same as the Python's on 33 parity cases, compared as JSON with key order. It walks the sample folder once instead of running a `glob` for each candidate path, and it reads step 33's table once for its two sections. On a synthetic 1,000,000-record VCF the variant count takes 0.05 s and 7 MB, where the Python collector took 0.9 s and 38 MB. What differs from the Python: - A file named `.vcf.gz` that is plain text is read. The Python marked the variants section unreadable. - A sample name with `*`, `?` or `[` is matched as written. The Python read it as a file pattern. diff --git a/bin/cn_loci.tsv b/bin/cn_loci.tsv index aa1b5d4c6..6f63e7cf7 100644 --- a/bin/cn_loci.tsv +++ b/bin/cn_loci.tsv @@ -66,7 +66,7 @@ HP gene chr16 HP_UNIT1:72056529-72057466:HP exons 3 and 4, the first copy of the LCE3BC interval chr1 LCE3BC_DEL:152600233-152614098:LCE3C to LCE3B, inside the LCE3C_LCE3B deletion LCE3_FLANK_L:152520000-152560000:flank on the LCE3D side, outside the deletion;LCE3_FLANK_R:152640000-152680000:flank on the LCE3A side, outside the deletion . sum flanks 2 0 .,-Q 1 . . 8 1.5 round:0.3 . 0=LCE3B and LCE3C deleted on both chromosomes (LCE3C_LCE3B-del on both), a common genotype that goes with a higher psoriasis risk|1=LCE3B and LCE3C deleted on one chromosome (LCE3C_LCE3B-del on one)|2=no LCE3B/LCE3C deletion seen|more={n} copies of the LCE3B/LCE3C stretch: more than two, a duplication bedcov cn-depth none 19169253 CCL3L1 gene chr17 CCL3L3:36194868-36196758:CCL3L3, the one copy of CCL3L1/CCL3L3 on the reference (ENST00000619989) CCL3L_FLANK_L:36112000-36148000:single-copy sequence between the CCL3 and CCL3L3 duplications;CCL3L_FLANK_R:36460000-36500000:single-copy sequence past the duplications . sum flanks 2 0 .,-Q 1 . . 8 1.5 round:0.3 . 0=no copy of CCL3L1 or CCL3L3|1=one copy of CCL3L1/CCL3L3 in all|more={n} copies of CCL3L1/CCL3L3 in all: the count differs widely between people, and no reading of it is given bedcov cn-depth none 15637236 CYP21A2 gene chr6 CYP21A2_BODY:32038950-32041644:CYP21A2 from the end of the C4A model to its last base (NM_000500.9);CYP21A1P_BODY:32006220-32008909:the same stretch of CYP21A1P (NR_040090.1) RCCX_FLANK_L:31945000-31975000:single-copy sequence before the RCCX duplication;RCCX_FLANK_R:32070000-32100000:single-copy sequence in TNXB past the duplication PSV_In2G:32039080-32039081:32006352 AC>G c.293-13A/C>G (In2G, intron 2);PSV_I172N:32039425-32039426:32006689 T>A c.518T>A (p.I172N, exon 4);PSV_V237E:32039809-32039810:32007073 T>A c.713T>A (p.V237E, exon 6 cluster);PSV_V281L:32040109-32040110:32007373 G>T c.844G>T (p.V281L, exon 7);PSV_Q318X:32040420-32040421:32007685 C>T c.955C>T (p.Q318X, exon 8);PSV_R356W:32040534-32040535:32007799 C>T c.1069C>T (p.R356W, exon 8) sum flanks 2 0 .,-Q 1 0.02 . 8 1.5 psv:0.4 . none=no sign of the 30 kb chimeric deletion or of a pseudogene conversion at the six sites read; short reads see neither a change elsewhere in CYP21A2 nor two copies on one chromosome and none on the other|deletion=one RCCX module fewer and one CYP21A2 copy fewer at the differing sites: the pattern of the 30 kb CYP21A1P/CYP21A2 chimeric deletion, a common cause of carrier status for 21-hydroxylase deficiency (congenital adrenal hyperplasia); unconfirmed: a clinical test (MLPA or long-range PCR) confirms it|conversion=fewer CYP21A2 copies carry the gene's own base at {sites} than at the other sites: a possible conversion of one CYP21A2 copy to the pseudogene sequence there, a carrier pattern for 21-hydroxylase deficiency (congenital adrenal hyperplasia) when it sits on one chromosome; which chromosome carries it is not known; unconfirmed: a clinical test confirms it|pseudogene_deletion=one RCCX module fewer with CYP21A2 kept at every site read: a CYP21A1P (pseudogene) deletion, common and not a carrier finding bedcov cn-depth paraphase:rccx 34285390,27228465 -# --- interval rows: written by scripts/ci/build-cn-intervals.py; do not edit by hand --- +# --- interval rows: written by scripts/ci/build-cn-intervals.sh; do not edit by hand --- 1q21.1_TAR_syndrome_proximal_BP2-BP3 interval chr1 1q21.1_TAR_syndrome_proximal_BP2-BP3_t1:145686999-145809382:1q21.1 recurrent (TAR syndrome) region (proximal, BP2-BP3) (includes RBM8A);1q21.1_TAR_syndrome_proximal_BP2-BP3_t2:145823828-145923649;1q21.1_TAR_syndrome_proximal_BP2-BP3_t3:145927476-146048495 1q21.1_TAR_syndrome_proximal_BP2-BP3_L:143034587-143134587:flank left of the region;1q21.1_TAR_syndrome_proximal_BP2-BP3_R:149926380-150026380:flank right of the region . pooled flanks 2 0 . . . 8 1.5 round:0.3 . . bedcov cn-depth cnvpytor:cnvs . 1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4 interval chr1 1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_t1:145687000-145809382:1q21.1 recurrent (TAR syndrome) region (proximal and distal, BP1-BP4) (includes RBM8A and GJA5);1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_t2:145823828-145923649;1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_t3:145927476-146048496;1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_t4:147105903-147473859;1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_t5:147473859-147841815;1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_t6:147842996-147917509 1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_L:143034587-143134587:flank left of the region;1q21.1_TAR_syndrome_proximal_and_distal_BP1-BP4_R:149926380-150026380:flank right of the region . pooled flanks 2 0 . . . 8 1.5 round:0.3 . . bedcov cn-depth cnvpytor:cnvs . 1q21.1_distal_BP3-BP4 interval chr1 1q21.1_distal_BP3-BP4_t1:147105904-147473860:1q21.1 recurrent region (distal, BP3-BP4) (includes GJA5);1q21.1_distal_BP3-BP4_t2:147473860-147841815;1q21.1_distal_BP3-BP4_t3:147842996-147917509 1q21.1_distal_BP3-BP4_L:143034587-143134587:flank left of the region;1q21.1_distal_BP3-BP4_R:149926380-150026380:flank right of the region . pooled flanks 2 0 . . . 8 1.5 round:0.3 . . bedcov cn-depth cnvpytor:cnvs . diff --git a/bin/copy_number.py b/bin/copy_number.py index 2e1b95485..9412104b9 100755 --- a/bin/copy_number.py +++ b/bin/copy_number.py @@ -31,7 +31,7 @@ "call", "interval", "readings", "input", "owner", "cross_check", "pmids"] KINDS = ("gene", "interval", "organelle") # GRCh38 names of the primary chromosomes (the ClinGen BED spells chrX 'chrx': -# scripts/ci/build-cn-intervals.py normalises it, and a row with any other +# og dev build cn-intervals normalises it, and a row with any other # spelling is refused here). CHROMS = {f"chr{c}" for c in list(range(1, 23)) + ["X", "Y", "M"]} INPUTS = ("bedcov", "cnvpytor-level") diff --git a/docs/cn-loci.md b/docs/cn-loci.md index eb6f0a44e..a9b918f5f 100644 --- a/docs/cn-loci.md +++ b/docs/cn-loci.md @@ -89,7 +89,7 @@ A site is low when its estimate is more than 0.6 copies under the larger of the ### The interval rows -`scripts/ci/build-cn-intervals.py` writes these rows, everything in `bin/cn_loci.tsv` after its marker line; do not edit them by hand. Step 35 (and the PARASCOPY module) runs `og cn bed --kind interval`, one `samtools bedcov` over every segment of every row, then `og cn intervals`, which writes one line per row to `_cn_intervals.tsv`: locus, name, chrom, start, end, status, copies, expected, change (`deletion`, `duplication`, `no change`, or `.` when not called), estimate and reason. The step log lists the rows with a change. The rows give a total copy count only, never breakpoints. +`scripts/ci/build-cn-intervals.sh` writes these rows with `og dev build cn-intervals`, everything in `bin/cn_loci.tsv` after its marker line; do not edit them by hand. Step 35 (and the PARASCOPY module) runs `og cn bed --kind interval`, one `samtools bedcov` over every segment of every row, then `og cn intervals`, which writes one line per row to `_cn_intervals.tsv`: locus, name, chrom, start, end, status, copies, expected, change (`deletion`, `duplication`, `no change`, or `.` when not called), estimate and reason. The step log lists the rows with a change. The rows give a total copy count only, never breakpoints. - **The ClinGen recurrent regions**: every region row of ClinGen Dosage Sensitivity's recurrent CNV BED, version 2.1, GRCh38 (CC0, pinned by `CLINGEN_RECURRENT_CNV_URL` and its sha256 in `versions.env`). The file's breakpoint and LCR rows are the segmental duplications that bound the regions. They are cut out of the targets, as are GIAB v3.6 segmental duplications, assembly gaps, satellites and the regions of every pinned Parascopy model (CHRNA7 in 15q13.3 stays with Parascopy). Targets are cut to pieces of at most 500 kb, and their depth is pooled. Controls are two 100 kb windows, one on each side of the cluster of ClinGen rows the region belongs to, clear of all of that and of the targets of the hand rows below (so no row's control sits on another row's target); a side with no clear window within 3 Mb (a chromosome end, as next to 22q11.2 and 15q11.2) is left out. The BED spells its chrX rows `chrx`; the script normalises them, and the engine refuses any contig name that is not a GRCh38 chromosome. The classification of a CNV found here follows the ACMG and ClinGen technical standards (Riggs 2020, PMID 31690835); a call from this step is a lead for a clinical test, not a classification. - **Two more ClinGen curations**: the 1p36 terminal region (ISCA-37434, haploinsufficiency score 3) and the Xq28 region that holds MECP2 (ISCA-46304, triplosensitivity score 3: the MECP2 duplication), from ClinGen's region curation list, read on 2026-11-04. @@ -100,7 +100,7 @@ A row on chrX or chrY is read against the sample's own count there: its controls The couple screen (step 39) reads the table with `couple_report.py screen --cn-intervals ID=_cn_intervals.tsv`: each carrier allele is present, absent or not assessable for each partner, and a male partner's AZF deletions are listed the same way (present means the deletion is there). With `--cnvpytor ID=_cnvs.vcf.gz` (step 18), a CNVpytor call that disagrees with the depth call is printed beside it. Step 39 and the Nextflow couple screen pass both when the partner has them. Both reports show the table in step 35's section: the counts by status and each row with a deletion or a duplication. -To change a row, change the script (its hand rows or a pinned download) and run `scripts/ci/build-cn-intervals.py`; `--check` fails when the catalog is not what the downloads give. +To change a row, change its rule or hand row in `rust/crates/og-dev/src/build/cn_intervals.rs`, or a pinned download in `scripts/ci/build-cn-intervals.sh`, rebuild og (`scripts/setup.sh --og`) and run `scripts/ci/build-cn-intervals.sh`; `--check` fails when the catalog is not what the downloads give. ## One owner per locus @@ -118,7 +118,7 @@ pypgx owns the genes it calls from the BAM, the `bam` rows of `assets/pgx_genes. 1. Check that no pinned caller owns the gene: Parascopy (`assets/parascopy_models_v1.7.txt`), pypgx, Paraphase. 2. Choose single-copy controls near the gene, outside any common deletion or duplication there and outside segmental duplications. -3. Add the row to `bin/cn_loci.tsv`, above its marker line, and cite the method in `pmids`. An interval row (everything after the marker) is never added by hand: add it to `scripts/ci/build-cn-intervals.py` and rerun the script. +3. Add the row to `bin/cn_loci.tsv`, above its marker line, and cite the method in `pmids`. An interval row (everything after the marker) is never added by hand: add it to the hand rows of `rust/crates/og-dev/src/build/cn_intervals.rs` and rerun `scripts/ci/build-cn-intervals.sh`. 4. Run it from a step: `og cn bed --locus NAME`, `samtools bedcov` with the row's passes, then `og cn call --locus NAME --bedcov ... --table ...`. 5. Add made-up bedcov tables for the row to `tests/test_copy_number.py`, at every count it can call and at each guard. diff --git a/docs/rust.md b/docs/rust.md index cb6924559..ae50a8042 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -37,7 +37,7 @@ bin/og the shim tasks call; runs bin/.og/og-linux- (og-da tests/rust/ the differential test against CPython, the image test ``` -Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`; `clip_check.py` keeps only the names `collect_summary.py` imports), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). `cnvpytor_to_vcf.py` keeps only the Python copy of that rule, which step 39's couple screen still imports, and `annotsv_postfilter.py` only the column names and the consensus join the reports import. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (the commands of `chip_report.py`, which keeps only the detection limit and the column names the reports import). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, the commands of `copy_number.py` (which keeps the catalog reader the catalog tests import) and the marker reader of `traits_report.py` (which keeps `MARKERS` for the summary). Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`; `bin/ancestry_fine_report.py` keeps only the names `collect_summary.py` and the docs demo import. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`, which keeps the CYP2D6 and HLA rules the reports import), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`, which keeps the SMN1 rule beside Parascopy). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)); the file stays one release as a shim that runs it, for crontabs. `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. +Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`; `clip_check.py` keeps only the names `collect_summary.py` imports), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). `cnvpytor_to_vcf.py` keeps only the Python copy of that rule, which step 39's couple screen still imports, and `annotsv_postfilter.py` only the column names and the consensus join the reports import. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (the commands of `chip_report.py`, which keeps only the detection limit and the column names the reports import). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, the commands of `copy_number.py` (which keeps the catalog reader the catalog tests import) and the marker reader of `traits_report.py` (which keeps `MARKERS` for the summary). Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`; `bin/ancestry_fine_report.py` keeps only the names `collect_summary.py` and the docs demo import. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`, which keeps the CYP2D6 and HLA rules the reports import), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`, which keeps the SMN1 rule beside Parascopy). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)); the file stays one release as a shim that runs it, for crontabs. `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. `og dev build cn-intervals|callability|sv-rule-regions` rebuild the committed tables `bin/cn_loci.tsv` (its interval rows), `assets/callability_genes.tsv` and `assets/sv_rule_regions.tsv`; they replaced `scripts/ci/build-cn-intervals.py` and the Python inside `build-callability.sh` and `build-sv-rule-regions.sh`. Those scripts keep the downloads (`curl` through `scripts/ci/lib/fetch-pinned.sh`, each file checked against its sha256) and hand `og` the folder, so `og dev build` reads local files only. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. A Python file whose tool moved to `og` but whose names other Python still imports stays as a *remnant*: a `# remnant:` header names its importers and the lane that deletes it, and `tests/parity/remnants.tsv` lists the names it may hold and the files that import it. `og dev remnants` (Guard, job `rust`) fails when the file defines a name the list does not allow, when an allowed name has no user left, when a listed importer no longer imports it, or when an unlisted file starts to. So a remnant only shrinks. `bin/family_report.py` is the first one: the pedigree, kinship and PAR rules that `couple_report.py` and `pedigree_svg.py` import. `bin/genome_lookup.py` keeps the query reading and the summary names rule that `results_index.py` imports. diff --git a/docs/testing.md b/docs/testing.md index 7cd0a5da2..7302f594c 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -102,7 +102,7 @@ The base leg runs every base case whenever any case runs, because every shard st The images the cases use are cached in four GitHub Actions cache entries, one per group. `scripts/ci/e2e-run.sh --image-groups` names each image's group, taken from its repository name without the tag, so a version bump changes the key of one group only; a group whose key missed restores the newest older entry of that group and pulls only what changed. A group keeps its smallest images up to 1500 MB of zstd output, and the larger ones are pulled on every run. Four groups of 1500 MB and the six data entries below stay under the repository's 10 GB cache limit even while main replaces one group: 5.9 GB of images, 1.5 GB more for the group being replaced, and about 2.4 GB of data. Images pinned by digest are not cached, because `docker load` gives them back without the digest, and they are left out of the group's key, so a digest bump misses no entry. Only the base leg saves, and only on main (or in a run by hand with `save_images`, on any branch), after it pulled every image the cases use and got through its cases, and only the groups whose key missed: a cache saved by a pull request can be read by that pull request alone and would only push main's entries out. The job summary lists every image with its size and what the cache did with it: restored on an exact key hit, restored from an older entry, saved, or pulled. Every merge in the e2e scope starts a run on main, so the cache follows main's images. GitHub drops an entry no run has read for 7 days, so after a quiet week the next run on main fills the cache again. Until this layout the cache was one entry, and by October 2026 it had grown to about 11 GB, over the limit, so every run missed it. -Six downloads are cached the same way. Two come from EBI's 1000 Genomes server, which is slow and at times stalls: MoChA's chr20 panel with its manifest (0.64 GB, case `mocha-1`), keyed on `MOCHA_PANEL_MANIFEST_SHA256` and the chromosome, and the 1000 Genomes phase 3 sites VCF (1.36 GB, case `archaic-ancestry-1`), keyed on `ARCHAIC_1KG_SITES_MD5`. On a miss, each of those two is tried first at `E2E_DATA_MIRROR`, the GitHub release `e2e-data-1` that holds copies of the pinned files, and then at EBI; the pin checks both. The release also holds the four PGS Catalog score files, the HAPNEST panel and the table builders' downloads below, so an EBI outage costs none of them on a cold cache. It holds the TRExplorer catalog of `build-sv-rule-regions.sh` too (the v2.1 file its `TRX_SHA256` pins), so a miss tries the mirror first and the catalog's own GitHub release after it. Two are small and failed whole runs on 2026-11-04, because their case is in the base leg or a shard that cannot start without it: `hla.dat` of the two IPD-IMGT/HLA releases `bash-step-08-hla` installs, from raw.githubusercontent.com (which served it under 1 KB/s), with GENCODE's gene and exon files from EBI's FTP server (curl exit 7 and 28), and the four real PGS Catalog score files of `prs-scores-and-run-options-1-strand-sex`, from the same server. `scripts/ci/e2e-run.sh --data-key` names their keys: the releases and a hash of the code in `scripts/lib/common.sh` that installs them, and a hash of the score ids and their server. `scripts/ci/e2e-run.sh` hands each case its files just before it runs, at the paths `setup.sh` and `install_data_file` read them from, so they download nothing; the score files the case copies from the cache itself. On a miss it downloads them first, checked against their pins or the checksum files the servers publish beside them (`MD5SUMS`, `md5checksum.txt`, the `.md5` of each score), trying up to 6 times 30 s apart. The four score files are pinned by md5 in `PGS_MD5` of `e2e-run.sh` (the md5 the PGS Catalog served beside each on 2026-11-06), tried on the mirror first and then at EBI; a restored or downloaded score must match its pin as well as its `.md5`. A file gets at most `E2E_MIRROR_BUDGET` seconds on the mirror (900, or half the fill's budget when that is less), so a mirror that trickles leaves EBI its turn. One fill may take at most `E2E_DATA_BUDGET` seconds (2400 in `e2e.yml`): a source that trickles above `fetch`'s stall limit would otherwise run on to the leg's 240-minute limit, as EBI's 1000 Genomes server did twice on 2026-11-05. A fill still running then is stopped and its case fails as `FAIL (data fill over 2400s)` without running, since its own download would be as slow. A restored entry is checked again before use (`--data-check`): the IPD-IMGT/HLA and GENCODE entry against the sha256 list written when it was filled, the score files against their `.md5`; one that fails is downloaded again. `bash-step-08-hla` then runs with every HTTP and HTTPS request sent to a closed port, after a control shows that a request under that block fails at the closed port, so a warm cache cannot reach either host. Only the leg whose cases include the case restores and saves its entry, and it saves only on main or in a run by hand with `save_images`, as with the images. The job summary says whether each key hit, and the case's log says where its files came from. The fifth is the PGS Catalog's synthetic HAPNEST panel (0.27 GB, cases `prs-2-ancestry` and `prs-scores-and-run-options-2-panel-chrx`) from EBI's FTP server, which failed cases with curl exit 7 on 2026-11-04. It is keyed on the sha256 `tests/e2e/lib.sh` pins (`HAPNEST_SHA256`). The cases handle it themselves (`setup_ancestry_panel` in `lib.sh`): with the panel in the cache, or on a miss from `E2E_DATA_MIRROR`, they check the sha256 and run `setup.sh --ancestry-panel` with every HTTP and HTTPS request sent to a closed port, so an EBI outage cannot fail them; when neither has it, `setup.sh` downloads the panel from EBI as a user's run does. The first case puts a panel that did not come from the cache in the cache for the save step. A red control in `prs-2-ancestry` runs `setup.sh` with the network blocked and no panel, which must fail. The sixth, entry `catalogs`, holds the pinned downloads of `scripts/ci/build-callability.sh`, `scripts/ci/build-cn-intervals.py` and `scripts/ci/build-sv-rule-regions.sh`, whose cases `callability-1-catalog`, `cn-intervals-1-catalog` and `followups-ci-3-sv-rule-regions` rebuild `assets/callability_genes.tsv`, the interval rows of `bin/cn_loci.tsv` and `assets/sv_rule_regions.tsv` and compare them with the committed files. Each builder lists its downloads with `--list`; the key is a hash of those lists, so a new pin is a new key. GENCODE's GTF among them comes from EBI's FTP server, which refused every connection for minutes on 2026-11-06 and failed `callability-1-catalog` on main (run 42259432681). The runner fills the entry as it does the others, mirror first and every file checked against its sha256, and each case points its builder's cache at its folder there; a builder that still has to download tries for about five minutes, 30 s apart. Every download, there and in `setup.sh`, gives up on a transfer that stalls (`FETCH_STALL`, see `fetch` in `scripts/lib/common.sh`), and the next try resumes it. +Six downloads are cached the same way. Two come from EBI's 1000 Genomes server, which is slow and at times stalls: MoChA's chr20 panel with its manifest (0.64 GB, case `mocha-1`), keyed on `MOCHA_PANEL_MANIFEST_SHA256` and the chromosome, and the 1000 Genomes phase 3 sites VCF (1.36 GB, case `archaic-ancestry-1`), keyed on `ARCHAIC_1KG_SITES_MD5`. On a miss, each of those two is tried first at `E2E_DATA_MIRROR`, the GitHub release `e2e-data-1` that holds copies of the pinned files, and then at EBI; the pin checks both. The release also holds the four PGS Catalog score files, the HAPNEST panel and the table builders' downloads below, so an EBI outage costs none of them on a cold cache. It holds the TRExplorer catalog of `build-sv-rule-regions.sh` too (the v2.1 file its `TRX_SHA256` pins), so a miss tries the mirror first and the catalog's own GitHub release after it. Two are small and failed whole runs on 2026-11-04, because their case is in the base leg or a shard that cannot start without it: `hla.dat` of the two IPD-IMGT/HLA releases `bash-step-08-hla` installs, from raw.githubusercontent.com (which served it under 1 KB/s), with GENCODE's gene and exon files from EBI's FTP server (curl exit 7 and 28), and the four real PGS Catalog score files of `prs-scores-and-run-options-1-strand-sex`, from the same server. `scripts/ci/e2e-run.sh --data-key` names their keys: the releases and a hash of the code in `scripts/lib/common.sh` that installs them, and a hash of the score ids and their server. `scripts/ci/e2e-run.sh` hands each case its files just before it runs, at the paths `setup.sh` and `install_data_file` read them from, so they download nothing; the score files the case copies from the cache itself. On a miss it downloads them first, checked against their pins or the checksum files the servers publish beside them (`MD5SUMS`, `md5checksum.txt`, the `.md5` of each score), trying up to 6 times 30 s apart. The four score files are pinned by md5 in `PGS_MD5` of `e2e-run.sh` (the md5 the PGS Catalog served beside each on 2026-11-06), tried on the mirror first and then at EBI; a restored or downloaded score must match its pin as well as its `.md5`. A file gets at most `E2E_MIRROR_BUDGET` seconds on the mirror (900, or half the fill's budget when that is less), so a mirror that trickles leaves EBI its turn. One fill may take at most `E2E_DATA_BUDGET` seconds (2400 in `e2e.yml`): a source that trickles above `fetch`'s stall limit would otherwise run on to the leg's 240-minute limit, as EBI's 1000 Genomes server did twice on 2026-11-05. A fill still running then is stopped and its case fails as `FAIL (data fill over 2400s)` without running, since its own download would be as slow. A restored entry is checked again before use (`--data-check`): the IPD-IMGT/HLA and GENCODE entry against the sha256 list written when it was filled, the score files against their `.md5`; one that fails is downloaded again. `bash-step-08-hla` then runs with every HTTP and HTTPS request sent to a closed port, after a control shows that a request under that block fails at the closed port, so a warm cache cannot reach either host. Only the leg whose cases include the case restores and saves its entry, and it saves only on main or in a run by hand with `save_images`, as with the images. The job summary says whether each key hit, and the case's log says where its files came from. The fifth is the PGS Catalog's synthetic HAPNEST panel (0.27 GB, cases `prs-2-ancestry` and `prs-scores-and-run-options-2-panel-chrx`) from EBI's FTP server, which failed cases with curl exit 7 on 2026-11-04. It is keyed on the sha256 `tests/e2e/lib.sh` pins (`HAPNEST_SHA256`). The cases handle it themselves (`setup_ancestry_panel` in `lib.sh`): with the panel in the cache, or on a miss from `E2E_DATA_MIRROR`, they check the sha256 and run `setup.sh --ancestry-panel` with every HTTP and HTTPS request sent to a closed port, so an EBI outage cannot fail them; when neither has it, `setup.sh` downloads the panel from EBI as a user's run does. The first case puts a panel that did not come from the cache in the cache for the save step. A red control in `prs-2-ancestry` runs `setup.sh` with the network blocked and no panel, which must fail. The sixth, entry `catalogs`, holds the pinned downloads of `scripts/ci/build-callability.sh`, `scripts/ci/build-cn-intervals.sh` and `scripts/ci/build-sv-rule-regions.sh`, whose cases `callability-1-catalog`, `cn-intervals-1-catalog` and `followups-ci-3-sv-rule-regions` rebuild `assets/callability_genes.tsv`, the interval rows of `bin/cn_loci.tsv` and `assets/sv_rule_regions.tsv` and compare them with the committed files. Each builder lists its downloads with `--list`; the key is a hash of those lists, so a new pin is a new key. GENCODE's GTF among them comes from EBI's FTP server, which refused every connection for minutes on 2026-11-06 and failed `callability-1-catalog` on main (run 42259432681). The runner fills the entry as it does the others, mirror first and every file checked against its sha256, and each case points its builder's cache at its folder there; a builder that still has to download tries for about five minutes, 30 s apart, and gives up at once on a 403 or 404 (`fetch_pinned` in `scripts/ci/lib/fetch-pinned.sh`, which the three builders share). The builders download with `curl` and hand the folder to `og dev build`, which reads local files only. Every download, there and in `setup.sh`, gives up on a transfer that stalls (`FETCH_STALL`, see `fetch` in `scripts/lib/common.sh`), and the next try resumes it. How long it took in one job, before the split: a passing e2e job took about 105 to 205 minutes with 84 cases, and 230 minutes of cases with 130 (run 41390434697, 2026-11-02). The runner decides much of it, not the cache: the same CPU model took 106 minutes in one run and 166 in another. In October 2026, with 84 cases, the same code took 106 and 166 minutes on an AMD EPYC 9V45 (runs 39197502832 and 39075176392, the second of which also saved the cache), 112 minutes on an Intel Xeon 6973P (run 39088086725), 156 and 191 minutes on an AMD EPYC 7763 (runs 39075764278 and 39075201465) and 205 minutes on an AMD EPYC 9V74 (run 39197512594, which saved three groups). The image cache saves about 7 minutes of that: pulling every image took 9 minutes, restoring and loading the four groups took 2.5 (run 39075201465 restored 39 images in 118 seconds), and saving them all took 12. Each leg has a limit of 240 minutes (355 for a traced shard leg); the base leg waits up to 75 minutes for a fixture a pull request bumps. Each leg's first steps print the runner's CPU model and CPU count to the log and the job summary, so a later timing can name the hardware. diff --git a/scripts/ci/build-callability.sh b/scripts/ci/build-callability.sh index e47058489..ab594545b 100755 --- a/scripts/ci/build-callability.sh +++ b/scripts/ci/build-callability.sh @@ -8,9 +8,8 @@ # scripts/ci/build-callability.sh --list the downloads, one "NAME URL SHA256" per line # # Downloads seven pinned files (about 85 MB, each checked against its -# sha256) into a temporary folder, or reads them from CALLABILITY_CACHE when -# that folder holds them with the right sha256, writes OUT and deletes the -# downloads: +# sha256) into CALLABILITY_CACHE, where a file with the right sha256 is read +# as it is, or else into a temporary folder that is deleted, and writes OUT: # # GIAB CMRG v1.00's GRCh38 gene coordinates (Wagner 2022, PMID 35132260): # the 273 challenging medically relevant genes @@ -53,7 +52,9 @@ # NEB, PIK3CA, PLG, VWF). Step 35's own report section reads all of them. # tests/test_paralog_loci.py holds that list and goes red when it changes. # -# Needs curl, python3 and sha256sum or shasum. The e2e case +# The rules are in og dev build callability +# (rust/crates/og-dev/src/build/callability.rs). Needs curl, og +# (scripts/setup.sh --og) and sha256sum or shasum. The e2e case # tests/e2e/callability-1-catalog.sh runs it and fails on any difference with # the committed file. set -euo pipefail @@ -81,208 +82,17 @@ if [ "${1:-}" = --list ]; then exit 0 fi -sha256() { - if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | cut -d' ' -f1; else shasum -a 256 "$1" | cut -d' ' -f1; fi -} - -WORK=$(mktemp -d) -trap 'rm -rf "$WORK"' EXIT - -# get NAME URL SHA256: WORK/NAME from CALLABILITY_CACHE when it matches, else -# downloaded; stop unless it matches. -get() { - local cached="${CALLABILITY_CACHE:-}/$1" got - if [ -n "${CALLABILITY_CACHE:-}" ] && [ -f "$cached" ] && [ "$(sha256 "$cached")" = "$3" ]; then - cp "$cached" "${WORK}/$1" - return - fi - echo "Downloading $2" >&2 - # Tries spread over about five minutes: EBI's FTP server refused every - # connection for minutes at a time (curl exit 7, 2026-11-06). - curl -fsSL --retry 6 --retry-all-errors --retry-delay 30 --retry-max-time 300 -o "${WORK}/$1" "$2" - got=$(sha256 "${WORK}/$1") - if [ "$got" != "$3" ]; then - echo "ERROR: $1 has sha256 ${got}, want $3" >&2 - exit 1 - fi - if [ -n "${CALLABILITY_CACHE:-}" ]; then - mkdir -p "$CALLABILITY_CACHE" && cp "${WORK}/$1" "$cached" - fi -} +# shellcheck source=scripts/ci/lib/fetch-pinned.sh +. "${REPO}/scripts/ci/lib/fetch-pinned.sh" +DIR=${CALLABILITY_CACHE:-} +if [ -z "$DIR" ]; then + DIR=$(mktemp -d) + trap 'rm -rf "$DIR"' EXIT +fi for f in "${FILES[@]}"; do IFS='|' read -r name url sum <<<"$f" - get "$name" "$url" "$sum" + fetch_pinned "$DIR" "$name" "$url" "$sum" || exit 1 done -python3 - "$WORK" "$REPO" "${OUT}.part" <<'PY' -import csv -import gzip -import re -import sys -from collections import defaultdict - -work, repo, out = sys.argv[1:4] -CHROMS = [f"chr{i}" for i in range(1, 23)] + ["chrX", "chrY"] -ORDER = {c: i for i, c in enumerate(CHROMS)} -PRIMARY = set(CHROMS) | {"chrM"} -ATT = re.compile(r'(\S+) "([^"]*)"') -SEGDUP_SHARE = 0.5 -REASONS = ("CMRG", "segdup", "false_duplication", "collapsed", "ALT-only") - - -def bed(name): - d = defaultdict(list) - with (gzip.open if name.endswith(".gz") else open)(f"{work}/{name}", "rt") as f: - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) >= 3 and not line.startswith(("#", "track", "browser")): - d[c[0]].append((int(c[1]), int(c[2]))) - for k in d: - d[k].sort() - return d - - -def covered(chrom, s, e, d): - """Bases of [s, e) inside the intervals of d (merged as it goes).""" - n, cur = 0, s - for a, b in d.get(chrom, []): - if b <= cur: - continue - if a >= e: - break - a, b = max(a, cur), min(b, e) - if b > a: - n += b - a - cur = b - return n - - -# ALT scaffolds: GenBank accession -> UCSC name (GENCODE names them by accession) -alt = {} -with open(f"{work}/report.txt") as f: - for line in f: - c = line.rstrip("\n").split("\t") - if not line.startswith("#") and len(c) >= 10 and c[1] == "alt-scaffold": - alt[c[4]] = c[9] - -genes = {} # hgnc id -> [symbol, chrom, start0, end] (any gene type) -coding = set() # hgnc ids of the protein-coding ones -on_primary = set() # every gene symbol on chr1-22, X, Y or M -alt_genes = {} # hgnc id -> [symbol, UCSC ALT name] (first ALT copy) -with gzip.open(f"{work}/gencode.gtf.gz", "rt") as f: - for line in f: - if line.startswith("#"): - continue - c = line.split("\t", 9) - if c[2] != "gene": - continue - a = dict(ATT.findall(c[8])) - if c[0] in PRIMARY: - on_primary.add(a.get("gene_name")) - h = a.get("hgnc_id") - if not h or a["gene_id"].endswith("_PAR_Y"): - continue - if c[0] in ORDER and h not in genes: - genes[h] = [a["gene_name"], c[0], int(c[3]) - 1, int(c[4])] - if a.get("gene_type") == "protein_coding": - coding.add(h) - elif c[0] in alt and h not in alt_genes and a.get("gene_type") == "protein_coding": - alt_genes[h] = [a["gene_name"], alt[c[0]]] - -reasons = defaultdict(set) -by_symbol = {g[0]: h for h, g in genes.items()} -with open(f"{work}/cmrg.bed") as f: - for line in f: - chrom, s, e, sym = line.rstrip("\n").split("\t")[:4] - s, e = int(s), int(e) - h = by_symbol.get(sym) - if h is None: - # renamed since CMRG v1.00: the protein-coding gene that covers most of its span - best = max(((min(e, genes[k][3]) - max(s, genes[k][2]), k) for k in coding if genes[k][1] == chrom), - default=(0, None)) - if best[0] <= 0: - sys.exit(f"ERROR: CMRG gene {sym} ({chrom}:{s}-{e}) matches no GENCODE 50 gene") - h = best[1] - reasons[h].add("CMRG") - -segdups, collapsed = bed("segdups.bed.gz"), bed("collapsed.bed.gz") -falsedup = bed("falsedup_correct.bed.gz") -for k, v in bed("falsedup_incorrect.bed.gz").items(): - falsedup[k] = sorted(falsedup[k] + v) -for h in coding: - sym, chrom, s, e = genes[h] - if covered(chrom, s, e, segdups) >= SEGDUP_SHARE * (e - s): - reasons[h].add("segdup") - if covered(chrom, s, e, falsedup): - reasons[h].add("false_duplication") - if covered(chrom, s, e, collapsed): - reasons[h].add("collapsed") -for h, (sym, name) in alt_genes.items(): - if h not in genes and sym not in on_primary: - reasons[h].add("ALT-only") - - -def table(path): - with open(path) as f: - return list(csv.DictReader((x for x in f if x.strip() and not x.startswith("#")), delimiter="\t", - quoting=csv.QUOTE_NONE)) - - -parascopy = {g for r in table(f"{repo}/assets/paralog_loci.tsv") for g in r["genes"].split(",")} -# A gene step 32 skips on the default no-ALT reference (callable_without_alt no) has no -# caller there, and step 32 refuses a BAM with ALT contigs, so it is not pypgx's. -pypgx = {r["gene"] for r in table(f"{repo}/assets/pgx_genes.tsv") - if r["mode"] == "bam" and r["callable_without_alt"] != "no"} -# A cn_loci gene row reports the genes it names (its locus, as RHD, or LPA for LPA_KIV2, -# or a target, as HBA1 and HBA2), not every gene whose span holds one of its targets. -cn_names = {} -for r in table(f"{repo}/bin/cn_loci.tsv"): - if r["kind"] == "gene": - names = {r["locus"], r["locus"].split("_")[0]} - for part in r["targets"].split(";"): - names |= {part.split(":")[0], part.split(":")[0].split("_")[0]} - for n in names: - cn_names.setdefault(n, r["locus"]) - - -def owner(sym): - if sym in parascopy: - return "parascopy" - if sym in pypgx: - return "pypgx" - if sym in cn_names: - return f"cn_loci:{cn_names[sym]}" - return "." - - -rows = [] -for h, why in reasons.items(): - if h in genes: - sym, chrom, s, e = genes[h] - rows.append((ORDER[chrom], s, sym, h, chrom, s, e, why, owner(sym))) - else: - sym, name = alt_genes[h] - rows.append((len(ORDER), 0, sym, h, name, ".", ".", why, owner(sym))) -rows.sort() -syms = [r[2] for r in rows] -dup = sorted({x for x in syms if syms.count(x) > 1}) -if dup: - sys.exit(f"ERROR: gene symbols listed twice: {', '.join(dup)}") -counts = {k: sum(1 for r in rows if k in r[7]) for k in REASONS} -owned = sum(1 for r in rows if r[8] != ".") -with open(out, "w") as f: - f.write("# assets/callability_genes.tsv: the genes where short reads are often not placed with confidence, so a\n" - "# 'no finding' there needs a reading of its own. Step 16b measures each gene on chr1-22, X and Y; the\n" - "# report lists the ones it could not read. Written by scripts/ci/build-callability.sh, which says how the\n" - "# rows and the owners are chosen; rebuild it there, do not edit it by hand. start is 0-based and end\n" - "# 1-based, as in a BED file; an ALT-only gene has its ALT scaffold as chrom and '.' as start and end.\n" - f"# Genes: {len(rows)} (" + ", ".join(f"{k} {counts[k]}" for k in REASONS) + f"); with an owner: {owned}.\n") - f.write("gene\thgnc_id\tchrom\tstart\tend\treasons\towner\n") - for r in rows: - f.write("\t".join(map(str, (r[2], r[3], r[4], r[5], r[6], ",".join(k for k in REASONS if k in r[7]), r[8]))) - + "\n") -print(f"Genes: {len(rows)} (" + ", ".join(f"{k} {counts[k]}" for k in REASONS) + f"); with an owner: {owned}", - file=sys.stderr) -PY -mv "${OUT}.part" "$OUT" +"${REPO}/bin/og" dev build callability --work "$DIR" --repo "$REPO" --out "$OUT" echo "Wrote ${OUT}" >&2 diff --git a/scripts/ci/build-cn-intervals.py b/scripts/ci/build-cn-intervals.py deleted file mode 100755 index 5982debdf..000000000 --- a/scripts/ci/build-cn-intervals.py +++ /dev/null @@ -1,423 +0,0 @@ -#!/usr/bin/env python3 -"""build-cn-intervals.py: write the interval rows of bin/cn_loci.tsv. - -The interval rows are the copy-number checks step 35 runs with one -`samtools bedcov` pass (og cn intervals): - - - every region row of ClinGen Dosage Sensitivity's recurrent CNV BED, - version 2.1, GRCh38 (CLINGEN_RECURRENT_CNV_URL and _SHA256 in - versions.env, CC0): 22q11.2, 1q21.1, 15q11.2, 16p11.2, 17p12, 17p11.2, - 7q11.23, 3q29, Xp22.31 and the rest. The file's breakpoint and LCR rows - (itemRgb 0,0,0) are the segmental duplications that bound the regions: - they are cut out of the targets and kept out of the controls. Its chrX - rows are spelled 'chrx' and are normalised to chrX; any other name that is - not a GRCh38 chromosome stops the build; - - two ClinGen curations the BED lacks (HAND_ROWS): the 1p36 terminal - region (ISCA-37434, haploinsufficiency 3) and the Xq28 region that holds - MECP2 (ISCA-46304, triplosensitivity 3), from - ClinGen_region_curation_list_GRCh38.tsv as read on 2026-11-04; - - three carrier deletion alleles at their ClinVar GRCh38 spans (HAND_ROWS): - del(GJB6-D13S1830) (VCV 5546), CFTRdele2,3 (VCV 66105) and a 621 bp HBB - deletion that removes exon 3 (VCV 3769040); and two beta-globin - deletions at their HbVar GRCh38 spans, checked on 2026-11-04 by placing - 50 bp on each side of the NG_000007.3 breakpoints on NC_000011.10: the - 619 bp HBB deletion common in India and Pakistan (HbVar 979, - NG_000007.3:g.71609_72227del619, Orkin 1979) and the 13.4 kb Sicilian - (delta-beta)0 deletion from HBD to 3' of HBB (HbVar 1035, - NG_000007.3:g.64336_77738del13403, Henthorn 1990). The alpha-globin - --SEA deletion is read by the HBA row (step 35's HBA table), not here; - - the Y chromosome AZF regions (HAND_ROWS), read only on a sample with a Y: - AZFa as USP9Y and DDX3Y, AZFb as KDM5D and EIF1AY (single-copy genes each - deletion removes), and AZFc as the four DAZ copies added up (a complete - AZFc deletion leaves none, the partial gr/gr and b2/b3 deletions two; - which copies are lost, and the breakpoints, are not claimed). Gene spans - from Ensembl's REST lookup, checked on 2026-11-04. Controls: ZFY and - NLGN4Y, single-copy genes outside every AZF region. - -Targets are cut to pieces of at most 500 kb and lose every stretch inside a -breakpoint or LCR row of the BED, a GIAB v3.6 segmental duplication, an -assembly gap (GIAB's gaps with 15 kb on each side), a satellite or a region of -a model the pinned Parascopy data holds (assets/parascopy_models_v1.7.txt: -Parascopy owns those, CHRNA7 in 15q13.3 among them); the AZF targets keep -their segmental duplications, since the DAZ copies are one. Controls are two -100 kb windows, one on each side of the cluster of ClinGen rows the region -belongs to, the nearest that stay clear of all of that, within 3 Mb; a side -with none (a chromosome end) is left out. Every row: depth pooled over the -targets against depth pooled over the controls, scale 2, a whole number -within 0.3 (round:0.3), at least 8x of control depth (4x on chrY), the two -flanks within 1.5x of each other; cross_check cnvpytor:cnvs (step 18). - -Usage: - scripts/ci/build-cn-intervals.py [--catalog bin/cn_loci.tsv] [--cache DIR] [--check] - scripts/ci/build-cn-intervals.py --list the downloads, one "NAME URL SHA256" per line - -Rewrites the catalog's rows from the marker line on (MARKER), or with --check -exits 1 when the catalog is not what the downloads give. The downloads (about -150 KB) are checked against their sha256 and kept in --cache when given. The -e2e case tests/e2e/cn-intervals-1-catalog.sh runs --check. -""" -import argparse -import gzip -import hashlib -import os -import re -import sys -import tempfile -import time -import urllib.request - -REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -MARKER = "# --- interval rows: written by scripts/ci/build-cn-intervals.py; do not edit by hand ---" -GIAB = "https://ftp-trace.ncbi.nlm.nih.gov/ReferenceSamples/giab/release/genome-stratifications/v3.6/GRCh38@all" -MASKS = [ - ("segdups.bed.gz", f"{GIAB}/SegmentalDuplications/GRCh38_segdups.bed.gz", - "09ac029a7d0d681a9f0ed6581a6bc10fc708a834bc3f91c46a9bef504f6055c5"), - ("gaps.bed.gz", f"{GIAB}/OtherDifficult/GRCh38_gaps_slop15kb.bed.gz", - "b5fdbfc7e5b9cf4fa356f23841fd7b8a74fa239fd8cbc5b5c3463c9e7bdc0902"), - ("satellites.bed.gz", f"{GIAB}/LowComplexity/GRCh38_satellites_slop5.bed.gz", - "10207aa2dac0e1189ea553a374abbdcb4de25a81378d88dc4ba03aea8718b5a2"), -] -REGION_RGB, MASK_RGB = "255,111,0", "0,0,0" -CHROMS = {f"chr{c}" for c in list(range(1, 23)) + ["X", "Y"]} -# GRCh38 chromosome lengths (the no-ALT analysis set's .fai). -LENGTHS = {"chr1": 248956422, "chr2": 242193529, "chr3": 198295559, "chr4": 190214555, "chr5": 181538259, - "chr6": 170805979, "chr7": 159345973, "chr8": 145138636, "chr9": 138394717, "chr10": 133797422, - "chr11": 135086622, "chr12": 133275309, "chr13": 114364328, "chr14": 107043718, "chr15": 101991189, - "chr16": 90338345, "chr17": 83257441, "chr18": 80373285, "chr19": 58617616, "chr20": 64444167, - "chr21": 46709983, "chr22": 50818468, "chrX": 156040895, "chrY": 57227415} -PIECE, FLANK, GAP, SEARCH, STEP, MIN_PIECE = 500_000, 100_000, 50_000, 3_000_000, 10_000, 500 - -# Rows the BED lacks: locus, chrom, [(name, 0-based start, end)] targets, [controls] or None (flanks), -# target_depth, whether segmental duplications are cut out of the targets, pmids. The label of the -# first target is the row's name in step 35's table. -Y_CONTROLS = [("ZFY", 2935200, 2982579, "ZFY, single-copy control on Yp"), - ("NLGN4Y", 14522572, 14845654, "NLGN4Y, single-copy control on Yq")] -HAND_ROWS = [ - ("1p36_terminal", "chr1", [("1p36 terminal region (includes GABRD), ClinGen ISCA-37434", 898702, 6229913)], - None, "pooled", True, "."), - ("Xq28_MECP2", "chrX", [("Xq28 region (includes MECP2), ClinGen ISCA-46304: the MECP2 duplication", - 154008528, 154110279)], None, "pooled", True, "."), - ("GJB6_D13S1830", "chr13", [("del(GJB6-D13S1830), ClinVar 5546", 20223037, 20531806)], None, "pooled", True, - "11807148"), - ("CFTR_dele2_3", "chr7", [("CFTRdele2,3 (21 kb), ClinVar 66105", 117498312, 117519392)], None, "pooled", True, - "."), - ("HBB_exon3_del", "chr11", [("HBB deletion of exon 3 (621 bp), ClinVar 3769040", 5225254, 5225875)], None, - "pooled", True, "."), - ("HBB_619bp_del", "chr11", [("HBB 619 bp deletion (South Asian), HbVar 979", 5225388, 5226007)], None, - "pooled", True, "287080"), - ("HBD_HBB_Sicilian_del", "chr11", [("Sicilian (delta-beta)0 deletion (13.4 kb, HBD to HBB 3'), HbVar 1035", - 5219877, 5233280)], None, "pooled", True, "2307466"), - ("AZFa", "chrY", [("AZFa (USP9Y and DDX3Y)", 12537649, 12860852), ("DDX3Y", 12904107, 12920478)], - Y_CONTROLS, "pooled", False, "37674303"), - ("AZFb", "chrY", [("AZFb (KDM5D and EIF1AY)", 19702297, 19745148), ("EIF1AY", 20575693, 20593154)], - Y_CONTROLS, "pooled", False, "37674303"), - ("AZFc_DAZ", "chrY", [("AZFc (DAZ1 to DAZ4 added up; gr/gr and b2/b3 leave two)", 23129354, 23199010), - ("DAZ2", 23219446, 23291356), ("DAZ3", 24763068, 24813492), ("DAZ4", 24833842, 24907040)], - Y_CONTROLS, "sum", False, "37674303"), -] -COLUMNS = ["locus", "kind", "chrom", "targets", "controls", "details", "target_depth", "control_depth", - "scale", "offset", "bedcov", "mapq0_min", "mapq0_max", "min_control_depth", "max_control_ratio", - "call", "interval", "readings", "input", "owner", "cross_check", "pmids"] - - -class BuildError(ValueError): - pass - - -def sha256(path): - h = hashlib.sha256() - with open(path, "rb") as f: - for b in iter(lambda: f.read(1 << 20), b""): - h.update(b) - return h.hexdigest() - - -def fetch(name, url, want, folder): - path = os.path.join(folder, name) - if not (os.path.exists(path) and sha256(path) == want): - for i in range(3): - try: - with urllib.request.urlopen(url, timeout=120) as r, open(path + ".part", "wb") as f: - f.write(r.read()) - os.replace(path + ".part", path) - break - except OSError as e: - # A missing or forbidden file stays so: no point waiting. - if i == 2 or getattr(e, "code", None) in (403, 404): - raise BuildError(f"cannot download {url}: {e}") - time.sleep(30) - got = sha256(path) - if got != want: - raise BuildError(f"{url}: sha256 {got}, pinned {want}") - return path - - -def versions_env(key): - with open(os.path.join(REPO, "versions.env")) as f: - for line in f: - m = re.match(rf'^{key}="([^"]*)"', line) - if m: - return m.group(1) - raise BuildError(f"no {key} in versions.env") - - -def norm_chrom(c): - """chrx -> chrX (the ClinGen BED's spelling); anything not a GRCh38 chromosome stops the build.""" - c2 = re.sub(r"^chr([xy])$", lambda m: "chr" + m.group(1).upper(), c) - if c2 not in CHROMS: - raise BuildError(f"chromosome {c!r} is not a GRCh38 chromosome name") - return c2 - - -def read_clingen(lines): - """(regions [(chrom, start, end, name)], masks [(chrom, start, end)]) of the BED's lines.""" - regions, masks = [], [] - for line in lines: - if not line.strip() or line.startswith(("#", "track", "browser")): - continue - c = line.rstrip("\n").split("\t") - if len(c) < 9: - raise BuildError(f"ClinGen BED: a row with {len(c)} columns: {line.strip()!r}") - chrom, s, e, name, rgb = norm_chrom(c[0]), int(c[1]), int(c[2]), c[3].strip(), c[8].strip() - if e <= s: - raise BuildError(f"ClinGen BED: {name}: empty span") - if rgb == REGION_RGB: - regions.append((chrom, s, e, name)) - elif rgb == MASK_RGB: - masks.append((chrom, s, e)) - else: - raise BuildError(f"ClinGen BED: {name}: itemRgb {rgb!r} is neither a region nor a breakpoint row") - if not regions: - raise BuildError("ClinGen BED: no region row") - return regions, masks - - -def read_bed(path): - out = [] - with gzip.open(path, "rt") as f: - for line in f: - if line.startswith("#") or not line.strip(): - continue - c = line.split("\t") - out.append((c[0], int(c[1]), int(c[2]))) - return out - - -def read_parascopy(path): - out = [] - with open(path) as f: - for line in f: - if line.startswith("#") or not line.strip(): - continue - _, reg, _ = line.split() - c, span = reg.split(":") - s, e = span.split("-") - out.append((c, int(s) - 1, int(e))) - return out - - -def merge(spans): - out = [] - for s, e in sorted(spans): - if out and s <= out[-1][1]: - out[-1][1] = max(out[-1][1], e) - else: - out.append([s, e]) - return [tuple(x) for x in out] - - -def subtract(s, e, cuts): - """[(s, e)] of s-e outside the merged spans cuts.""" - out, pos = [], s - for cs, ce in cuts: - if ce <= pos or cs >= e: - continue - if cs > pos: - out.append((pos, cs)) - pos = max(pos, ce) - if pos < e: - out.append((pos, e)) - return out - - -def pieces(spans): - out = [] - for s, e in spans: - if e - s < MIN_PIECE: - continue - n = -(-(e - s) // PIECE) - step = -(-(e - s) // n) - for i in range(n): - out.append((s + i * step, min(e, s + (i + 1) * step))) - return out - - -def clear(s, e, cuts): - return all(ce <= s or cs >= e for cs, ce in cuts) - - -def flank(chrom, edge, side, cuts): - """The nearest FLANK window GAP beyond edge on side (-1 left, +1 right), clear of cuts, within SEARCH.""" - for k in range(0, SEARCH // STEP + 1): - if side < 0: - e = edge - GAP - k * STEP - s = e - FLANK - else: - s = edge + GAP + k * STEP - e = s + FLANK - if s < 0 or e > LENGTHS[chrom]: - return None - if clear(s, e, cuts): - return (s, e) - return None - - -def locus_id(name): - band = name.split()[0] - groups = [g for g in re.findall(r"\(([^()]*)\)", name) if not g.strip().startswith("includes")] - words = re.sub(r"[^A-Za-z0-9.-]+", "_", " ".join(groups)).strip("_") - return f"{band}_{words}" if words else band - - -def label(text): - return re.sub(r"[;\t\n]", ",", text).strip() - - -def build(clingen_lines, segdups, gaps, sats, parascopy): - regions, masks = read_clingen(clingen_lines) - by_chrom = {} - for name, items in (("mask", masks), ("segdup", segdups), ("gap", gaps), ("sat", sats), ("para", parascopy)): - for c, s, e in items: - by_chrom.setdefault(c, {}).setdefault(name, []).append((s, e)) - # Clusters: ClinGen rows (regions and breakpoints) that touch or overlap, per chromosome. - clusters = {} - for c in CHROMS: - spans = [(s, e + 1) for cc, s, e, _ in regions if cc == c] + [(s, e + 1) for cc, s, e in masks if cc == c] - clusters[c] = merge(spans) - - def cuts(c, with_segdups=True): - d = by_chrom.get(c, {}) - keys = ("mask", "segdup", "gap", "sat", "para") if with_segdups else ("mask", "gap", "para") - return merge([x for k in keys for x in d.get(k, [])]) - - def flank_cuts(c): - # A control never sits on a target: the ClinGen regions and the HAND_ROWS targets are cut out too. - d = by_chrom.get(c, {}) - return merge([x for k in ("mask", "segdup", "gap", "sat", "para") for x in d.get(k, [])] - + [(s, e) for cc, s, e, _ in regions if cc == c] - + [(s, e) for _, cc, tl, *_ in HAND_ROWS if cc == c for _, s, e in tl]) - - rows, seen = [], {} - - def add(locus, chrom, targets, controls, target_depth, pmids): - if not targets: - raise BuildError(f"{locus}: no target stretch is left outside the masks") - if locus in seen: - seen[locus] += 1 - locus = f"{locus}_{seen[locus]}" - else: - seen[locus] = 1 - y = chrom == "chrY" - tg = ";".join(f"{locus}_t{i + 1}:{s}-{e}" + (f":{lab}" if lab else "") for i, (s, e, lab) in enumerate(targets)) - ct = ";".join(f"{locus}_{n}:{s}-{e}:{lab}" for n, s, e, lab in controls) - rows.append({"locus": locus, "kind": "interval", "chrom": chrom, "targets": tg, "controls": ct, - "details": ".", "target_depth": target_depth, "control_depth": "pooled" if y else "flanks", - "scale": "2", "offset": "0", "bedcov": ".", "mapq0_min": ".", "mapq0_max": ".", - "min_control_depth": "4" if y else "8", "max_control_ratio": "." if y else "1.5", - "call": "round:0.3", "interval": ".", "readings": ".", "input": "bedcov", "owner": "cn-depth", - "cross_check": "cnvpytor:cnvs", "pmids": pmids}) - - def flanks(chrom, s, e): - cs, ce = next(((a, b) for a, b in clusters[chrom] if a <= s and e <= b), (s, e)) - fc = flank_cuts(chrom) - out = [] - for side, edge, n in ((-1, cs, "L"), (1, ce, "R")): - w = flank(chrom, edge, side, fc) - if w: - out.append((n, w[0], w[1], "flank " + ("left" if side < 0 else "right") + " of the region")) - if not out: - raise BuildError(f"{chrom}:{s}-{e}: no flank clear of the masks within {SEARCH // 1_000_000} Mb") - return out - - for chrom, s, e, name in sorted(regions, key=lambda r: (sorted(CHROMS, key=_order).index(r[0]), r[1], r[2])): - segs = pieces(subtract(s, e, cuts(chrom))) - targets = [(a, b, label(name) if i == 0 else "") for i, (a, b) in enumerate(segs)] - add(locus_id(name), chrom, targets, flanks(chrom, s, e), "pooled", ".") - for locus, chrom, tlist, controls, td, with_sd, pmids in HAND_ROWS: - c = cuts(chrom, with_sd) - targets = [] - for name, s, e in tlist: - segs = pieces(subtract(s, e, c)) if td == "pooled" else [(s, e)] - if td == "sum" and not clear(s, e, merge(by_chrom.get(chrom, {}).get("para", []))): - raise BuildError(f"{locus}: {name} overlaps a Parascopy model") - targets += [(a, b, label(name) if i == 0 else "") for i, (a, b) in enumerate(segs)] - if controls is None: - ctl = flanks(chrom, tlist[0][1], tlist[-1][2]) - else: - ctl = [(f"c{i + 1}", s, e, lab) for i, (_, s, e, lab) in enumerate(controls)] - add(locus, chrom, targets, ctl, td, pmids) - return rows - - -def _order(c): - k = c[3:] - return (0, int(k)) if k.isdigit() else (1, k) - - -def render(rows): - return [MARKER + "\n"] + ["\t".join(r[c] for c in COLUMNS) + "\n" for r in rows] - - -def main(argv=None): - ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) - ap.add_argument("--catalog", default=os.path.join(REPO, "bin", "cn_loci.tsv")) - ap.add_argument("--cache") - ap.add_argument("--check", action="store_true") - ap.add_argument("--list", action="store_true") - a = ap.parse_args(argv) - if a.list and a.check: - ap.error("--list and --check do not go together") - if a.list: - try: - files = [("clingen_recurrent_cnv.bed", versions_env("CLINGEN_RECURRENT_CNV_URL"), - versions_env("CLINGEN_RECURRENT_CNV_SHA256"))] + MASKS - except BuildError as e: - print(f"ERROR: {e}", file=sys.stderr) - return 2 - for n, u, h in files: - print(n, u, h) - return 0 - try: - with tempfile.TemporaryDirectory() as tmp: - folder = a.cache or tmp - os.makedirs(folder, exist_ok=True) - bed = fetch("clingen_recurrent_cnv.bed", versions_env("CLINGEN_RECURRENT_CNV_URL"), - versions_env("CLINGEN_RECURRENT_CNV_SHA256"), folder) - sd, gp, st = (read_bed(fetch(n, u, h, folder)) for n, u, h in MASKS) - with open(bed, errors="replace") as f: - rows = build(f.readlines(), sd, gp, st, - read_parascopy(os.path.join(REPO, "assets", "parascopy_models_v1.7.txt"))) - except (OSError, BuildError) as e: - print(f"ERROR: {e}", file=sys.stderr) - return 2 - with open(a.catalog) as f: - lines = f.readlines() - keep = lines[:lines.index(MARKER + "\n")] if MARKER + "\n" in lines else lines - new = keep + render(rows) - if a.check: - if new != lines: - old = set(lines) - set(new) - print(f"{a.catalog} is not what the pinned downloads give ({len(old)} line(s) differ); " - "rerun scripts/ci/build-cn-intervals.py", file=sys.stderr) - return 1 - print(f"{a.catalog}: the {len(rows)} interval rows are what the pinned downloads give") - return 0 - with open(a.catalog + ".tmp", "w") as f: - f.writelines(new) - os.replace(a.catalog + ".tmp", a.catalog) - print(f"{a.catalog}: {len(rows)} interval rows written") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/ci/build-cn-intervals.sh b/scripts/ci/build-cn-intervals.sh new file mode 100755 index 000000000..ec40d052c --- /dev/null +++ b/scripts/ci/build-cn-intervals.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# build-cn-intervals.sh: rewrite the interval rows of bin/cn_loci.tsv, the +# copy-number checks step 35 runs with one samtools bedcov pass +# (og cn intervals). +# +# Usage: scripts/ci/build-cn-intervals.sh [--catalog FILE] [--cache DIR] [--check] +# scripts/ci/build-cn-intervals.sh --list the downloads, one "NAME URL SHA256" per line +# +# Rewrites FILE (default bin/cn_loci.tsv) from its marker line on, or with +# --check exits 1 when it is not what the downloads give. The four downloads +# (about 150 KB, each checked against its sha256) are kept in --cache when +# given, else in a temporary folder that is deleted: +# +# ClinGen Dosage Sensitivity's recurrent CNV BED, version 2.1, GRCh38 +# (CLINGEN_RECURRENT_CNV_URL and _SHA256 in versions.env, CC0) +# GIAB genome stratifications v3.6, GRCh38: segmental duplications, +# assembly gaps with 15 kb on each side, satellites +# +# with assets/parascopy_models_v1.7.txt. Every region row of the BED is a +# row, then the rows the BED lacks: 1p36 and Xq28 (MECP2) from ClinGen's +# curations, five carrier deletions (GJB6-D13S1830, CFTRdele2,3, three +# beta-globin deletions) and the Y chromosome's AZF regions. Targets are cut +# to pieces of at most 500 kb and lose the BED's breakpoint and LCR rows, +# segmental duplications (not on the AZF rows), gaps, satellites and +# Parascopy models; controls are two 100 kb flanks clear of all of that. The +# rules and the hand rows are in og dev build cn-intervals +# (rust/crates/og-dev/src/build/cn_intervals.rs); docs/cn-loci.md says what +# each row reads. +# +# Needs curl, og (scripts/setup.sh --og) and sha256sum or shasum. The e2e +# case tests/e2e/cn-intervals-1-catalog.sh runs --check. +set -euo pipefail + +REPO=$(cd "$(dirname "$0")/../.." && pwd) +# shellcheck source=scripts/ci/lib/fetch-pinned.sh +. "${REPO}/scripts/ci/lib/fetch-pinned.sh" + +CATALOG=${REPO}/bin/cn_loci.tsv +CACHE="" +CHECK="" +LIST="" +while [ $# -gt 0 ]; do + case $1 in + --catalog) CATALOG=${2:?--catalog needs a file}; shift 2 ;; + --cache) CACHE=${2:?--cache needs a folder}; shift 2 ;; + --check) CHECK=1; shift ;; + --list) LIST=1; shift ;; + *) echo "Usage: $0 [--catalog FILE] [--cache DIR] [--check] | --list" >&2; exit 2 ;; + esac +done +if [ -n "$LIST" ] && [ -n "$CHECK" ]; then + echo "ERROR: --list and --check do not go together" >&2 + exit 2 +fi + +# versions_env KEY: the value of KEY in versions.env. +versions_env() { + local v + v=$(sed -n "s/^$1=\"\([^\"]*\)\".*/\1/p" "${REPO}/versions.env" | head -n 1) + [ -n "$v" ] || { echo "ERROR: no $1 in versions.env" >&2; return 1; } + echo "$v" +} + +GIAB=https://ftp-trace.ncbi.nlm.nih.gov/ReferenceSamples/giab/release/genome-stratifications/v3.6/GRCh38@all +clingen_url=$(versions_env CLINGEN_RECURRENT_CNV_URL) || exit 2 +clingen_sha=$(versions_env CLINGEN_RECURRENT_CNV_SHA256) || exit 2 +FILES=( + "clingen_recurrent_cnv.bed|${clingen_url}|${clingen_sha}" + "segdups.bed.gz|${GIAB}/SegmentalDuplications/GRCh38_segdups.bed.gz|09ac029a7d0d681a9f0ed6581a6bc10fc708a834bc3f91c46a9bef504f6055c5" + "gaps.bed.gz|${GIAB}/OtherDifficult/GRCh38_gaps_slop15kb.bed.gz|b5fdbfc7e5b9cf4fa356f23841fd7b8a74fa239fd8cbc5b5c3463c9e7bdc0902" + "satellites.bed.gz|${GIAB}/LowComplexity/GRCh38_satellites_slop5.bed.gz|10207aa2dac0e1189ea553a374abbdcb4de25a81378d88dc4ba03aea8718b5a2" +) + +if [ -n "$LIST" ]; then + for f in "${FILES[@]}"; do + IFS='|' read -r name url sum <<< "$f" + echo "${name} ${url} ${sum}" + done + exit 0 +fi + +if [ -z "$CACHE" ]; then + CACHE=$(mktemp -d) + trap 'rm -rf "$CACHE"' EXIT +fi +for f in "${FILES[@]}"; do + IFS='|' read -r name url sum <<< "$f" + fetch_pinned "$CACHE" "$name" "$url" "$sum" || exit 2 +done + +# Not exec: the EXIT trap deletes a temporary download folder. +rc=0 +"${REPO}/bin/og" dev build cn-intervals \ + --clingen "${CACHE}/clingen_recurrent_cnv.bed" --segdups "${CACHE}/segdups.bed.gz" \ + --gaps "${CACHE}/gaps.bed.gz" --satellites "${CACHE}/satellites.bed.gz" \ + --parascopy "${REPO}/assets/parascopy_models_v1.7.txt" --catalog "$CATALOG" ${CHECK:+--check} || rc=$? +exit "$rc" diff --git a/scripts/ci/build-sv-rule-regions.sh b/scripts/ci/build-sv-rule-regions.sh index cb21651da..031b80fab 100755 --- a/scripts/ci/build-sv-rule-regions.sh +++ b/scripts/ci/build-sv-rule-regions.sh @@ -10,9 +10,8 @@ # "NAME URL SHA256" per line # # Downloads four pinned files (about 145 MB, each checked against its sha256) -# into a temporary folder, or reads them from SV_RULE_CACHE when that folder -# holds them with the right sha256 (and keeps what it downloads there), writes -# OUT and deletes the downloads: +# into SV_RULE_CACHE, where a file with the right sha256 is read as it is, or +# else into a temporary folder that is deleted, and writes OUT: # # TRExplorer v2.1's GRCh38 tandem repeat catalog (broadinstitute/trexplorer-catalog) # GENCODE 50's basic annotation (the release scripts/lib/common.sh pins) @@ -35,8 +34,12 @@ # GenCC disease curation; name ALIAS|PSEUDOGENE|its HGNC id| # the HGNC id of the gene that holds ALIAS as its symbol # -# Needs curl, python3 and sha256sum or shasum. It reads about 1 GB of text, -# so run it on a box with room, not in a unit test. +# A gene's span is that of the first GENCODE record of its HGNC id, and the +# _PAR_Y copies of the pseudoautosomal genes are skipped, as +# build-callability.sh does. The rules are in og dev build sv-rule-regions +# (rust/crates/og-dev/src/build/sv_rules.rs). Needs curl, og +# (scripts/setup.sh --og) and sha256sum or shasum. It reads about 3 GB of +# text in a few seconds and keeps little of it. set -euo pipefail REPO=$(cd "$(dirname "$0")/../.." && pwd) @@ -66,161 +69,19 @@ if [ "${1:-}" = --list ]; then exit 0 fi -sha256() { - if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | cut -d' ' -f1; else shasum -a 256 "$1" | cut -d' ' -f1; fi -} - +# shellcheck source=scripts/ci/lib/fetch-pinned.sh +. "${REPO}/scripts/ci/lib/fetch-pinned.sh" WORK=$(mktemp -d) trap 'rm -rf "$WORK"' EXIT +DIR=${SV_RULE_CACHE:-$WORK} [ -z "$CHECK" ] || OUT="${WORK}/rebuilt.tsv" -# get NAME URL SHA256: WORK/NAME from SV_RULE_CACHE when it matches, else -# downloaded; stop unless it matches. -get() { - local cached="${SV_RULE_CACHE:-}/$1" got - if [ -n "${SV_RULE_CACHE:-}" ] && [ -f "$cached" ] && [ "$(sha256 "$cached")" = "$3" ]; then - cp "$cached" "${WORK}/$1" - return - fi - echo "Downloading $2" >&2 - # Tries spread over about five minutes: EBI's FTP server refused every - # connection for minutes at a time (curl exit 7, 2026-11-06). - curl -fsSL --retry 6 --retry-all-errors --retry-delay 30 --retry-max-time 300 -o "${WORK}/$1" "$2" - got=$(sha256 "${WORK}/$1") - if [ "$got" != "$3" ]; then - echo "ERROR: $1 has sha256 ${got}, want $3" >&2 - exit 1 - fi - if [ -n "${SV_RULE_CACHE:-}" ]; then - mkdir -p "$SV_RULE_CACHE" && cp "${WORK}/$1" "$cached" - fi -} -get trx.bed.gz "$TRX_URL" "$TRX_SHA256" -get gencode.gtf.gz "$GENCODE_URL" "$GENCODE_SHA256" -get hgnc.txt "$HGNC_URL" "$HGNC_SHA256" -get regions.txt "$REGIONS_URL" "$REGIONS_SHA256" - -python3 - "$WORK" "${OUT}.part" <<'PY' -import bisect -import csv -import gzip -import re -import sys -from collections import defaultdict - -work, out = sys.argv[1:3] -CHROMS = [f"chr{i}" for i in range(1, 23)] + ["chrX", "chrY"] -ORDER = {c: i for i, c in enumerate(CHROMS)} -MIN_TR = 50 -LOCUS_GAP = 1_000_000 -ATT = re.compile(r'(\S+) "([^"]*)"') -SRC_TR = "TRExplorer v2.1; GENCODE 50 MANE Select CDS" -SRC_IG = "GENCODE 50 IG and TR genes" -SRC_MHC = "GRCh38 assembly regions (GCA_000001405.15)" -SRC_ALIAS = "HGNC 2026-10-06; GENCODE 50" - -rows = [] -gene_span = {} # HGNC id -> (chrom, start0, end) -ig = defaultdict(list) # locus -> [(chrom, start0, end)] -cds = defaultdict(list) # chrom -> [(start0, end, gene)] -with gzip.open(f"{work}/gencode.gtf.gz", "rt") as f: - for line in f: - if line.startswith("#"): - continue - c = line.rstrip("\n").split("\t") - if c[0] not in ORDER: - continue - start, end = int(c[3]) - 1, int(c[4]) - if c[2] == "gene": - a = dict(ATT.findall(c[8])) - if a.get("hgnc_id"): - gene_span[a["hgnc_id"]] = (c[0], start, end) - name, gtype = a.get("gene_name", ""), a.get("gene_type", "") - if re.match(r"(IG|TR)_", gtype) and re.match(r"(IG[HKL]|TR[ABDG])", name): - locus = "TRA" if name.startswith("TRD") else name[:3] - ig[locus].append((c[0], start, end)) - elif c[2] == "CDS" and 'tag "MANE_Select"' in c[8]: - cds[c[0]].append((start, end, dict(ATT.findall(c[8]))["gene_name"])) - -# tandem_repeat: catalog repeats of MIN_TR bp or more on a MANE Select coding exon -index = {} -for chrom, iv in cds.items(): - iv.sort() - starts = [s for s, _, _ in iv] - maxend, m = [], 0 - for _, e, _ in iv: - m = max(m, e) - maxend.append(m) - index[chrom] = (iv, starts, maxend) -with gzip.open(f"{work}/trx.bed.gz", "rt") as f: - for line in f: - c = line.rstrip("\n").split("\t") - if c[0] not in index: - continue - s, e = int(c[1]), int(c[2]) - if e - s < MIN_TR: - continue - iv, starts, maxend = index[c[0]] - j = bisect.bisect_left(starts, e) - 1 - genes = set() - while j >= 0 and maxend[j] > s: - if iv[j][1] > s: - genes.add(iv[j][2]) - j -= 1 - if genes: - motif = c[3] if len(c[3]) <= 12 else f"{len(c[3])}bp motif" - rows.append((c[0], s, e, "tandem_repeat", "class_3", f"{'/'.join(sorted(genes))}:{motif}", SRC_TR)) - -# ig_tr_locus: per locus the largest group of genes on one chromosome -for locus, genes in ig.items(): - groups = [] - for chrom, s, e in sorted(genes): - g = groups[-1] if groups else None - if g and g[0] == chrom and s - g[2] < LOCUS_GAP: - g[2] = max(g[2], e) - g[3] += 1 - else: - groups.append([chrom, s, e, 1]) - chrom, s, e, _ = max(groups, key=lambda g: g[3]) - rows.append((chrom, s, e, "ig_tr_locus", "class_3", locus, SRC_IG)) - -# mhc: the assembly report's region (1-based, inclusive) -with open(f"{work}/regions.txt") as f: - mhc = sorted({tuple(l.split("\t")[:4]) for l in f if l.startswith("MHC\t")}) -if len(mhc) != 1: - sys.exit(f"ERROR: want one MHC region in the assembly report, got {mhc}") -_, chrom, s, e = mhc[0] -rows.append((f"chr{chrom}", int(s) - 1, int(e), "mhc", "class_3", "MHC", SRC_MHC)) - -# gene_alias: a pseudogene that carries a disease gene's symbol as alias or previous symbol -with open(f"{work}/hgnc.txt", newline="") as f: - hgnc = list(csv.DictReader(f, delimiter="\t")) -approved = {r["symbol"]: r for r in hgnc} -for r in hgnc: - if r["locus_group"] != "pseudogene" or r["hgnc_id"] not in gene_span: - continue - for sym in sorted({x.strip('"') for x in (r["alias_symbol"] + "|" + r["prev_symbol"]).split("|")} - {""}): - d = approved.get(sym) - if d and d["hgnc_id"] != r["hgnc_id"] and d["gencc"]: - chrom, s, e = gene_span[r["hgnc_id"]] - rows.append((chrom, s, e, "gene_alias", "remap_gene", f"{sym}|{r['symbol']}|{r['hgnc_id']}|{d['hgnc_id']}", - SRC_ALIAS)) +fetch_pinned "$DIR" trx.bed.gz "$TRX_URL" "$TRX_SHA256" || exit 1 +fetch_pinned "$DIR" gencode.gtf.gz "$GENCODE_URL" "$GENCODE_SHA256" || exit 1 +fetch_pinned "$DIR" hgnc.txt "$HGNC_URL" "$HGNC_SHA256" || exit 1 +fetch_pinned "$DIR" regions.txt "$REGIONS_URL" "$REGIONS_SHA256" || exit 1 -rows.sort(key=lambda x: (ORDER[x[0]], x[1], x[2], x[3], x[5])) -counts = defaultdict(int) -for x in rows: - counts[x[3]] += 1 -with open(out, "w") as f: - f.write("# assets/sv_rule_regions.tsv: the GRCh38 regions where og sv postfilter overrides AnnotSV's class 4\n" - "# or 5. Written by scripts/ci/build-sv-rule-regions.sh, which says how each rule's rows are chosen;\n" - "# rebuild it there, do not edit it by hand. start is 0-based and end 1-based, as in a BED file.\n" - "# Rows: " + ", ".join(f"{k} {counts[k]}" for k in sorted(counts)) + ".\n") - f.write("#chrom\tstart\tend\trule\taction\tname\tsource\n") - for x in rows: - f.write("\t".join(map(str, x)) + "\n") -print("Rows: " + ", ".join(f"{k} {counts[k]}" for k in sorted(counts)), file=sys.stderr) -PY -mv "${OUT}.part" "$OUT" +"${REPO}/bin/og" dev build sv-rule-regions --work "$DIR" --out "$OUT" if [ -n "$CHECK" ]; then if cmp -s "$OUT" "$CHECK"; then echo "${CHECK} is what the pinned downloads give, byte for byte" >&2 diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index f76068dcf..fe64ffe70 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -62,7 +62,7 @@ bash-step-20-numt assets/callability_genes.tsv bin/collect_summary.py bin/traits bash-step-bash42 assets/construct_variants.tsv bin/clip_check.py bin/cpsr_sample_id bin/og scripts/03d-octopus.sh scripts/04a-tiddit.sh scripts/10-telomere-hunter.sh scripts/17-cpsr.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh bloodtype-1 assets/bloodtype_markers.tsv bin/cn_loci.tsv bin/og modules/local/bloodtype/* modules/local/vcf_precheck/* scripts/41-bloodtype.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh bumps-drop-in-baseline scripts/ci/fake-docker/lib.sh -callability-1-catalog assets/callability_genes.tsv assets/paralog_loci.tsv assets/pgx_genes.tsv bin/cn_loci.tsv bin/og scripts/ci/build-callability.sh scripts/ci/fake-docker/lib.sh +callability-1-catalog assets/callability_genes.tsv assets/paralog_loci.tsv assets/pgx_genes.tsv bin/cn_loci.tsv bin/og scripts/ci/build-callability.sh scripts/ci/fake-docker/lib.sh scripts/ci/lib/fetch-pinned.sh callability-2-genes assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16b-mosdepth.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh chip-clonal-1 assets/chip_genes.tsv bin/chip_report.py bin/og bin/render_report.py modules/local/chip/* modules/local/vcf_precheck/* scripts/49-chip.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh chip-mt-1-roundtrip assets/par_grch38.bed bin/og scripts/40-export-consumer.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz @@ -75,7 +75,7 @@ clinvar-combined-allele-1 assets/construct_variants.tsv assets/deep_intronic_sit clinvar-cpsr-quality-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv bin/clip_check.py bin/og bin/render_report.py scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh cn-engine-1-parity assets/bloodtype_markers.tsv bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py scripts/41-bloodtype.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh cn-guard-1-mapq0 bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py modules/local/parascopy/* scripts/35-paralogs.sh -cn-intervals-1-catalog bin/cn_loci.tsv bin/og scripts/ci/build-cn-intervals.py scripts/ci/fake-docker/lib.sh +cn-intervals-1-catalog assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/og scripts/ci/build-cn-intervals.sh scripts/ci/fake-docker/lib.sh scripts/ci/lib/fetch-pinned.sh cn-intervals-2-hg002 assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/copy_number.py bin/og bin/traits_report.py modules/local/deepvariant/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh cn-report-1-tables assets/carrier_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/paraphase_report.py bin/render_report.py bin/results_index.py bin/traits_report.py scripts/24-html-report.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh comphet-phasing-1-reads assets/construct_variants.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/og modules/local/slivar/* modules/local/vcf_precheck/* modules/local/vep/* scripts/13-vep-annotation.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/comphet/make_fixture.py @@ -91,7 +91,7 @@ expansions-and-rfc1-1-motif assets/str_extra_loci.json assets/stub/db/catalog.js families-file-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/couple_report.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/couple/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/families/stub_families.tsv family-joint-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/somalier/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/11-roh-analysis.sh scripts/33-sample-qc.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh fixitfelix-chr21-1 assets/callability_genes.tsv scripts/16b-mosdepth.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -followups-ci-3-sv-rule-regions assets/sv_rule_regions.tsv scripts/ci/build-sv-rule-regions.sh scripts/ci/fake-docker/lib.sh +followups-ci-3-sv-rule-regions assets/sv_rule_regions.tsv bin/og scripts/ci/build-sv-rule-regions.sh scripts/ci/fake-docker/lib.sh scripts/ci/lib/fetch-pinned.sh followups-clinvar-traits-1-rules-html assets/genotype_rules.tsv assets/traits.tsv bin/render_report.py bin/traits_report.py scripts/42-traits.sh scripts/ci/check-rule-alleles.sh followups-cn-1-couple-report assets/carrier_genes.tsv assets/par_grch38.bed assets/stub/pedigree.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/couple/* modules/local/html_report/* scripts/39-couple.sh scripts/ci/fake-docker/lib.sh followups-nf-1-nf-wiring assets/stub/* assets/test_samplesheet.csv assets/steps.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py bin/run_state.py bin/traits_report.py conf/test.config conf/test_all.config conf/containers.config modules/local/html_report/* modules/local/couple/* modules/local/parascopy/* modules/local/deepvariant/* modules/local/cpsr/* modules/local/clinvar_screen/* modules/local/locityper/* workflows/* tests/fixtures/consumer_export/sites.tsv.gz bin/og diff --git a/scripts/ci/e2e-run.sh b/scripts/ci/e2e-run.sh index 8d4bfe1c4..4a5a45800 100755 --- a/scripts/ci/e2e-run.sh +++ b/scripts/ci/e2e-run.sh @@ -196,7 +196,7 @@ PGS000066 b34f31b8b3a7dbbbb58882a70e14649d"} # EBI's FTP server, which refused every connection on 2026-11-06 (run # 42259432681). CATALOGS="callability bash ${REPO}/scripts/ci/build-callability.sh -cn-intervals python3 ${REPO}/scripts/ci/build-cn-intervals.py +cn-intervals bash ${REPO}/scripts/ci/build-cn-intervals.sh sv-rule-regions bash ${REPO}/scripts/ci/build-sv-rule-regions.sh" # catalog_files: "FOLDER NAME URL SHA256" of every download of CATALOGS. catalog_files() { diff --git a/scripts/ci/lib/fetch-pinned.sh b/scripts/ci/lib/fetch-pinned.sh new file mode 100644 index 000000000..ccdb82f01 --- /dev/null +++ b/scripts/ci/lib/fetch-pinned.sh @@ -0,0 +1,61 @@ +# shellcheck shell=bash +# fetch-pinned.sh: the one download rule of the asset builders in +# scripts/ci/ (build-callability.sh, build-cn-intervals.sh, +# build-sv-rule-regions.sh). Source it; it defines two functions. +# +# fetch_pinned DIR NAME URL SHA256 +# Makes DIR/NAME the file at URL with that sha256. A file already there +# with the right sha256 is kept, so DIR can be a cache that outlives the +# run. Otherwise it prints "Downloading URL" on stderr (the e2e cases +# count these lines), downloads to DIR/NAME.part and renames it when the +# sha256 matches. A 403 or 404 fails at once: a missing or forbidden file +# stays so. Any other failure is tried again 6 times, 30 s apart, for +# about five minutes: EBI's FTP server refused every connection for +# minutes at a time on 2026-11-06 (curl exit 7). Returns 1 with an ERROR +# line when the file cannot be had. +# +# pinned_sha256 FILE +# The sha256 of FILE, from sha256sum or shasum. +# +# og never downloads: the builders fetch here, then hand the folder to +# og dev build. + +pinned_sha256() { + if command -v sha256sum > /dev/null 2>&1; then + sha256sum "$1" | cut -d' ' -f1 + else + shasum -a 256 "$1" | cut -d' ' -f1 + fi +} + +fetch_pinned() { + local dir=$1 name=$2 url=$3 want=$4 file code got + file="${dir}/${name}" + if [ -f "$file" ] && [ "$(pinned_sha256 "$file")" = "$want" ]; then + return 0 + fi + mkdir -p "$dir" || return 1 + echo "Downloading ${url}" >&2 + if ! code=$(curl -fsSL -w '%{http_code}' -o "${file}.part" "$url"); then + case $code in + 403 | 404) + rm -f "${file}.part" + echo "ERROR: cannot download ${url}: HTTP ${code}" >&2 + return 1 + ;; + esac + if ! curl -fsSL --retry 6 --retry-all-errors --retry-delay 30 --retry-max-time 300 \ + -o "${file}.part" "$url"; then + rm -f "${file}.part" + echo "ERROR: cannot download ${url}" >&2 + return 1 + fi + fi + got=$(pinned_sha256 "${file}.part") + if [ "$got" != "$want" ]; then + rm -f "${file}.part" + echo "ERROR: ${name} has sha256 ${got}, want ${want}" >&2 + return 1 + fi + mv "${file}.part" "$file" +} diff --git a/tests/e2e/cn-intervals-1-catalog.sh b/tests/e2e/cn-intervals-1-catalog.sh index 5b0db18cb..dd2a6262f 100755 --- a/tests/e2e/cn-intervals-1-catalog.sh +++ b/tests/e2e/cn-intervals-1-catalog.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# The interval rows of bin/cn_loci.tsv are what scripts/ci/build-cn-intervals.py +# The interval rows of bin/cn_loci.tsv are what scripts/ci/build-cn-intervals.sh # writes from its pinned downloads (ClinGen's recurrent CNV BED V2.1 and three # GIAB v3.6 stratifications, each checked against its sha256): a hand edit of # an interval row fails here. Control: a copy of the catalog with one target @@ -11,9 +11,9 @@ CACHE="${CASE_TMP}/downloads" [ -z "${E2E_DATA_CACHE:-}" ] || CACHE="${E2E_DATA_CACHE}/catalogs/cn-intervals" mkdir -p "$CACHE" -echo "+ scripts/ci/build-cn-intervals.py --check" -python3 "${REPO}/scripts/ci/build-cn-intervals.py" --cache "$CACHE" --check 2>&1 | tee "$STEP_LOG" -check_eq "build-cn-intervals.py --check exits 0 on the committed catalog" "${PIPESTATUS[0]}" 0 +echo "+ scripts/ci/build-cn-intervals.sh --check" +bash "${REPO}/scripts/ci/build-cn-intervals.sh" --cache "$CACHE" --check 2>&1 | tee "$STEP_LOG" +check_eq "build-cn-intervals.sh --check exits 0 on the committed catalog" "${PIPESTATUS[0]}" 0 check "the log says all 90 interval rows were checked" grep -q "the 90 interval rows are what the pinned downloads give" "$STEP_LOG" # Control: one interval target moved by 1 kb @@ -25,7 +25,7 @@ awk -F'\t' -v OFS='\t' '$1 == "22q11.2_DGS_proximal_A-B" && !done { "${REPO}/bin/cn_loci.tsv" > "$EDITED" # shellcheck disable=SC2016 # the positional arguments of bash -c check "control: the edited copy differs from the catalog" bash -c '! cmp -s "$1" "$2"' _ "$EDITED" "${REPO}/bin/cn_loci.tsv" -python3 "${REPO}/scripts/ci/build-cn-intervals.py" --cache "$CACHE" --catalog "$EDITED" --check > "${CASE_TMP}/edited.log" 2>&1 +bash "${REPO}/scripts/ci/build-cn-intervals.sh" --cache "$CACHE" --catalog "$EDITED" --check > "${CASE_TMP}/edited.log" 2>&1 check_eq "control: --check fails (exit 1) on a catalog with one interval target moved" "$?" 1 cat "${CASE_TMP}/edited.log" diff --git a/tests/test_copy_number.py b/tests/test_copy_number.py index b00a00302..337dff70a 100644 --- a/tests/test_copy_number.py +++ b/tests/test_copy_number.py @@ -36,7 +36,7 @@ each, and a row shifted by 1 kb no longer matches its fixture; the PARASCOPY module runs the same list; the CCL3L1 reading names no disease (a planted 'HIV' is caught); - 9. the interval rows (scripts/ci/build-cn-intervals.py): a small ClinGen + 9. the interval rows (og dev build cn-intervals, with og): a small ClinGen BED with a 'chrx' row converts to rows on chrX that load, the same rows left as 'chrx' are refused, and an unknown contig stops the build; the committed rows are the ClinGen regions plus the named hand rows, with @@ -684,7 +684,6 @@ def dosage(cat, read): # Section 9: the interval rows. -BUILD = os.path.join(REPO, "scripts", "ci", "build-cn-intervals.py") SMALL_BED = """track name="test" #chrom\tstart\tend\tname\tscore\tstrand\tthickStart\tthickEnd\titemRgb chr22\t18159724\t18924717\tBP-A\t0\t.\t18159724\t18924717\t0,0,0 @@ -724,39 +723,49 @@ def depths(r, target=30.0, control=30.0): return {n: target if n in tg else control for *_, n in cn.regions(r)} -def intervals(cat, read): - import importlib.util - spec = importlib.util.spec_from_file_location("build_cn_intervals", BUILD) - bld = importlib.util.module_from_spec(spec) - spec.loader.exec_module(bld) - rows = bld.build(SMALL_BED.splitlines(True), [], [], [], []) - check("converter: the two region rows of the small BED, its chrx row on chrX", - [(r["locus"], r["chrom"]) for r in rows[:2]] == [("22q11.2_DGS_proximal_A-D", "chr22"), ("Xp22.31", "chrX")], - [(r["locus"], r["chrom"]) for r in rows]) +def build_rows(bed): + """og dev build cn-intervals --stdout on a ClinGen BED and no masks: (exit code, rows as dicts, stderr).""" with tempfile.TemporaryDirectory() as t: - p = os.path.join(t, "c.tsv") - write_catalog(p, rows) - try: - got = len(cn.load_catalog(p)) - except cn.CatalogError as e: - got = str(e) - check("converter: its rows load", got == len(rows), got) - write_catalog(p, [dict(r, chrom="chrx") if r["chrom"] == "chrX" else r for r in rows]) - try: - cn.load_catalog(p) - got = "loaded" - except cn.CatalogError as e: - got = str(e) - check("converter: the same rows left as 'chrx' are refused", "chrom 'chrx'" in got, got) - try: - bld.build(SMALL_BED.replace("chrx", "chrQ").splitlines(True), [], [], [], []) - got = "built" - except bld.BuildError as e: - got = str(e) - check("converter: an unknown contig stops the build", "'chrQ' is not a GRCh38 chromosome" in got, got) - check("converter: no target piece overlaps a breakpoint row", - all(not (s < 18924717 and 18159724 < e) and not (s < 21562827 and 21111384 < e) for r in rows[:1] - for _, s, e, _ in cn.parse_row(r)["_targets"])) + p = os.path.join(t, "clingen.bed") + with open(p, "w") as f: + f.write(bed) + out = subprocess.run([OG, "dev", "build", "cn-intervals", "--clingen", p, "--segdups", os.devnull, + "--gaps", os.devnull, "--satellites", os.devnull, "--parascopy", os.devnull, + "--stdout"], capture_output=True, text=True) + rows = [dict(zip(cn.COLUMNS, line.split("\t"))) for line in out.stdout.splitlines() if not line.startswith("#")] + return out.returncode, rows, out.stderr + + +def intervals(cat, read): + if not OG: + print("skip: OG is not set: the converter checks of og dev build cn-intervals did not run") + else: + code, rows, err = build_rows(SMALL_BED) + check("converter: the two region rows of the small BED, its chrx row on chrX", + code == 0 and [(r["locus"], r["chrom"]) for r in rows[:2]] + == [("22q11.2_DGS_proximal_A-D", "chr22"), ("Xp22.31", "chrX")], + (code, err, [(r["locus"], r["chrom"]) for r in rows])) + with tempfile.TemporaryDirectory() as t: + p = os.path.join(t, "c.tsv") + write_catalog(p, rows) + try: + got = len(cn.load_catalog(p)) + except cn.CatalogError as e: + got = str(e) + check("converter: its rows load", got == len(rows), got) + write_catalog(p, [dict(r, chrom="chrx") if r["chrom"] == "chrX" else r for r in rows]) + try: + cn.load_catalog(p) + got = "loaded" + except cn.CatalogError as e: + got = str(e) + check("converter: the same rows left as 'chrx' are refused", "chrom 'chrx'" in got, got) + code, _, err = build_rows(SMALL_BED.replace("chrx", "chrQ")) + check("converter: an unknown contig stops the build", + code == 2 and "'chrQ' is not a GRCh38 chromosome" in err, (code, err)) + check("converter: no target piece overlaps a breakpoint row", + all(not (s < 18924717 and 18159724 < e) and not (s < 21562827 and 21111384 < e) for r in rows[:1] + for _, s, e, _ in cn.parse_row(r)["_targets"])) iv = [r for r in cat.values() if r["kind"] == "interval" and r["_passes"] == ["."]] names = [r["locus"] for r in iv] diff --git a/tests/test_e2e_data_cache.sh b/tests/test_e2e_data_cache.sh index bfaf97a2d..e403f51ca 100755 --- a/tests/test_e2e_data_cache.sh +++ b/tests/test_e2e_data_cache.sh @@ -74,7 +74,7 @@ else echo "FAIL: the pgs key"; failed=1; fi if "$RUN" --data-key catalogs | grep -qE '^e2e-data-catalogs-[0-9a-f]{16}$'; then echo "PASS: the catalogs key is a hash" else echo "FAIL: the catalogs key"; failed=1; fi # Every row must be "NAME URL SHA256": a malformed row would reach the fill. -list=$( { bash "${REPO}/scripts/ci/build-callability.sh" --list; python3 "${REPO}/scripts/ci/build-cn-intervals.py" --list; bash "${REPO}/scripts/ci/build-sv-rule-regions.sh" --list; } ) +list=$( { bash "${REPO}/scripts/ci/build-callability.sh" --list; bash "${REPO}/scripts/ci/build-cn-intervals.sh" --list; bash "${REPO}/scripts/ci/build-sv-rule-regions.sh" --list; } ) n=$(awk 'NF == 3 && $2 ~ /^https:/ && $3 ~ /^[0-9a-f]{64}$/' <<< "$list" | wc -l | tr -d ' ') all=$(wc -l <<< "$list" | tr -d ' ') if [ "$n" = 15 ] && [ "$all" = 15 ]; then echo "PASS: the three builders list 15 pinned downloads (7, 4 and 4), every row well formed" diff --git a/versions.env b/versions.env index b67c1e177..304024130 100644 --- a/versions.env +++ b/versions.env @@ -407,7 +407,7 @@ EXOMISER_PHENOTYPE_SHA256="aa923530ca43cf5051ed2ba7b221bb73deb2b0dd66695b2ba6046 # The recurrent CNV regions of the interval rows of bin/cn_loci.tsv (step 35): # ClinGen Dosage Sensitivity's recurrent CNV BED, version 2.1, GRCh38 (CC0), # checked against the sha256 of the file as downloaded on 2026-11-04. -# scripts/ci/build-cn-intervals.py turns it into the rows; the e2e case +# scripts/ci/build-cn-intervals.sh turns it into the rows; the e2e case # tests/e2e/cn-intervals-1-catalog.sh rebuilds them and fails on any # difference. Change the two together, then rerun the script. CLINGEN_RECURRENT_CNV_URL="https://ftp.clinicalgenome.org/ClinGen_recurrent_CNV_V2.1-hg38.bed" From 1f2b7fcadaf904c7d42df9284c78f1e9b679dfe2 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:29:25 +0100 Subject: [PATCH 03/17] test(dev): plant three wrong builder rules to show the tests go red Red-first control, reverted in the next commit: callability keeps the _PAR_Y copies, the SV rule builder keeps the last record per HGNC id, and each cn-intervals piece ends one base short. Bead: pgp-bs9u.35 --- rust/crates/og-dev/src/build/callability.rs | 3 ++- rust/crates/og-dev/src/build/cn_intervals.rs | 3 ++- rust/crates/og-dev/src/build/sv_rules.rs | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/rust/crates/og-dev/src/build/callability.rs b/rust/crates/og-dev/src/build/callability.rs index d7ce4dece..556c65865 100644 --- a/rust/crates/og-dev/src/build/callability.rs +++ b/rust/crates/og-dev/src/build/callability.rs @@ -111,7 +111,8 @@ fn read_gtf(path: &Path, alt: &HashMap) -> Result Vec { // Ceiling divisions of positive numbers. let n = (len + PIECE - 1) / PIECE; let step = (len + n - 1) / n; - out.extend((0..n).map(|i| (s + i * step, e.min(s + (i + 1) * step)))); + // PLANTED, reverted in the next commit: each piece ends one base short. + out.extend((0..n).map(|i| (s + i * step, e.min(s + (i + 1) * step) - 1))); } out } diff --git a/rust/crates/og-dev/src/build/sv_rules.rs b/rust/crates/og-dev/src/build/sv_rules.rs index 7004f02b7..1007fc81b 100644 --- a/rust/crates/og-dev/src/build/sv_rules.rs +++ b/rust/crates/og-dev/src/build/sv_rules.rs @@ -99,9 +99,10 @@ fn read_gtf(path: &Path) -> Result { b"gene" => { let (start, end) = r.span()?; if let Some(h) = r.attr("hgnc_id")?.filter(|h| !h.is_empty()) - && !r.attr("gene_id")?.is_some_and(|id| id.ends_with("_PAR_Y")) + && !r.attr("gene_id")?.is_some_and(|id| id.ends_with("_PAR_X")) { - g.span.entry(h.to_owned()).or_insert((chrom, start, end)); + // PLANTED, reverted in the next commit: the last record wins. + g.span.insert(h.to_owned(), (chrom, start, end)); } let (name, ty) = ( r.attr("gene_name")?.unwrap_or(""), From fb4c96c80c57ca9a15888f15b5c92f7e720ca751 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:32:54 +0100 Subject: [PATCH 04/17] feat(summary): read the clinical sections in Rust (og summary collect --only b) ClinVar, the carrier list, PharmCAT, CPIC, pypgx, Cyrius, CPSR, the clinical filter with its ACMG SF tier, slivar and the cyp2d6 block, each input read once, beside the Python collector. No caller changes. Bead: pgp-bs9u.42 --- rust/Cargo.lock | 2 + rust/crates/og-summary/Cargo.toml | 4 +- rust/crates/og-summary/src/collect.rs | 21 +- rust/crates/og-summary/src/collect/acmg.rs | 871 ++++++++++++++++ rust/crates/og-summary/src/collect/assets.rs | 297 ++++++ rust/crates/og-summary/src/collect/carrier.rs | 146 +++ .../crates/og-summary/src/collect/clinical.rs | 208 ++++ rust/crates/og-summary/src/collect/clinvar.rs | 799 ++++++++++++++ rust/crates/og-summary/src/collect/cpsr.rs | 264 +++++ rust/crates/og-summary/src/collect/group_b.rs | 284 +++++ rust/crates/og-summary/src/collect/model.rs | 5 + rust/crates/og-summary/src/collect/pgx.rs | 618 +++++++++++ rust/crates/og-summary/src/collect/scan.rs | 19 + .../crates/og-summary/src/collect/sections.rs | 18 + rust/crates/og-summary/src/collect/text.rs | 195 ++++ .../crates/og-summary/src/collect/two_hits.rs | 980 ++++++++++++++++++ 16 files changed, 4727 insertions(+), 4 deletions(-) create mode 100644 rust/crates/og-summary/src/collect/acmg.rs create mode 100644 rust/crates/og-summary/src/collect/assets.rs create mode 100644 rust/crates/og-summary/src/collect/carrier.rs create mode 100644 rust/crates/og-summary/src/collect/clinical.rs create mode 100644 rust/crates/og-summary/src/collect/clinvar.rs create mode 100644 rust/crates/og-summary/src/collect/cpsr.rs create mode 100644 rust/crates/og-summary/src/collect/group_b.rs create mode 100644 rust/crates/og-summary/src/collect/pgx.rs create mode 100644 rust/crates/og-summary/src/collect/text.rs create mode 100644 rust/crates/og-summary/src/collect/two_hits.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 4d91a18ca..2edbd24e9 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -839,6 +839,8 @@ dependencies = [ "flate2", "indexmap", "memchr", + "og-bam", + "og-clinvar", "og-io", "og-steps", "pycompat", diff --git a/rust/crates/og-summary/Cargo.toml b/rust/crates/og-summary/Cargo.toml index 1a7fa879c..d17584417 100644 --- a/rust/crates/og-summary/Cargo.toml +++ b/rust/crates/og-summary/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "og-summary" -description = "The summary cluster: the step tools that turn a third-party tool's output into one of OwnGenome's tables (step 33's sample QC verdict, step 25's PRS table, step 20's chrM tables)" +description = "The summary cluster: the step tools that turn a third-party tool's output into one of OwnGenome's tables (step 33's sample QC verdict, step 25's PRS table, step 20's chrM tables), and og summary collect, summary.json section by section" version.workspace = true edition.workspace = true license.workspace = true @@ -9,6 +9,8 @@ publish.workspace = true [dependencies] pycompat.workspace = true og-io.workspace = true +og-bam.workspace = true +og-clinvar.workspace = true og-steps.workspace = true indexmap.workspace = true memchr.workspace = true diff --git a/rust/crates/og-summary/src/collect.rs b/rust/crates/og-summary/src/collect.rs index 0ae779232..cc5e827f3 100644 --- a/rust/crates/og-summary/src/collect.rs +++ b/rust/crates/og-summary/src/collect.rs @@ -16,10 +16,20 @@ //! sections are read. A reader's error stays in its section, which is //! then `unreadable` with a note that names the file. +pub mod acmg; +pub mod assets; +pub mod carrier; +pub mod clinical; +pub mod clinvar; +pub mod cpsr; +pub mod group_b; pub mod model; +pub mod pgx; pub mod scan; pub mod sections; +pub mod text; pub mod time; +pub mod two_hits; use crate::args::{self, Kind, Opt}; use model::{GROUP_A, Manifest, RunStatus, Section, Spec, Summary}; @@ -29,7 +39,7 @@ use sections::{Callability, Coverage, Data, Mosaic, ReadCount, SampleQc, SexChec use std::io::Write; use std::path::Path; -const USAGE: &str = "usage: og summary collect --sample SAMPLE --sample-dir SAMPLE_DIR --only a +const USAGE: &str = "usage: og summary collect --sample SAMPLE --sample-dir SAMPLE_DIR --only {a,b} --out OUT [--declared-sex DECLARED_SEX] [--assets ASSETS] "; @@ -39,7 +49,7 @@ const OPTS: &[Opt] = &[ Opt("--declared-sex", Kind::Value, false), Opt("--out", Kind::Value, true), Opt("--assets", Kind::Value, false), - Opt("--only", Kind::Choice(&["a"]), true), + Opt("--only", Kind::Choice(&["a", "b"]), true), ]; /// What one collect reads. @@ -75,7 +85,11 @@ pub fn cli(argv: &[String], out: &mut impl Write, err: &mut impl Write) -> u8 { // A part of the sections must never replace the sample's complete // summary.json, so there is no default path while `--only` exists. let target = a.text("--out").to_owned(); - let summary = match collect(&input) { + let read = match a.text("--only") { + "b" => group_b::collect, + _ => collect, + }; + let summary = match read(&input) { Ok(s) => s, Err(e) => return crate::finish(Err(e), err), }; @@ -195,6 +209,7 @@ pub fn collect(input: &Input) -> Result { manifest, declared_sex, sections, + cyp2d6: None, }) } diff --git a/rust/crates/og-summary/src/collect/acmg.rs b/rust/crates/og-summary/src/collect/acmg.rs new file mode 100644 index 000000000..70439bbd8 --- /dev/null +++ b/rust/crates/og-summary/src/collect/acmg.rs @@ -0,0 +1,871 @@ +//! The ACMG secondary-findings tier of the clinical section: the variants +//! that are secondary findings by the rule of their gene +//! (`assets/acmg_sf.tsv`, ACMG SF v3.3, Lee et al., Genet Med 2025). +//! +//! Three sources: step 06's ClinVar P/LP hits, the clinical filter's rare +//! HIGH-impact records (step 23, ranked MANE Select, then canonical, then +//! any transcript, as `og vcf csq-rank` ranks them) and step 05's class 4 +//! and 5 SVs in the genes whose rule allows them. A variant of an SF gene +//! that its rule does not report (one MUTYH variant, a heterozygous HFE +//! p.C282Y) is listed under `not_reported` with the reason. A called +//! deletion counts as the second hit of a gene whose rule needs two +//! ([`two_hits`](super::two_hits)), unless read-backed phase puts both on +//! one copy. + +use super::assets::{ACMG_SF_VERSION, Key, Rule, acmg_sf_rules, carrier_genes}; +use super::clinvar::{ + COMMON_AF, Floor, Hit, HitRecords, allele_copies, floor_values, obj, opt_int, opt_s, +}; +use super::text::{self, digits, format_map, gt_field, info_map}; +use super::two_hits::{DeletionInputs, Spans, Variant, gene_deletions, two_hits}; +use indexmap::IndexMap; +use pycompat::json::Value; +use pycompat::text::isdigit; +use std::collections::{HashMap, HashSet}; + +/// The genotype check's floors: step 06's GQ and DP, the alt read share a +/// heterozygote must lie within and a homozygous alt call must reach. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ClinicalFloor { + pub call: Floor, + pub het_ab: (f64, f64), + pub hom_ab: f64, +} + +impl Default for ClinicalFloor { + fn default() -> Self { + ClinicalFloor { + call: Floor::default(), + het_ab: (0.2, 0.8), + hom_ab: 0.8, + } + } +} + +/// The length of the `[0-9.]+` run at the start of `s`. +fn run(s: &str) -> usize { + s.bytes() + .take_while(|c| c.is_ascii_digit() || *c == b'.') + .count() +} + +/// `re.search(KEY([0-9.]+)-([0-9.]+))` (with `pair`) or `KEY([0-9.]+)`: +/// the first place the whole pattern matches. +fn search<'a>(line: &'a str, key: &str, pair: bool) -> Option<(&'a str, Option<&'a str>)> { + line.match_indices(key).find_map(|(i, _)| { + let rest = &line[i + key.len()..]; + let n = run(rest); + if n == 0 { + return None; + } + if !pair { + return Some((&rest[..n], None)); + } + let hi = rest[n..].strip_prefix('-')?; + let m = run(hi); + (m > 0).then(|| (&rest[..n], Some(&hi[..m]))) + }) +} + +fn float(s: &str) -> Result { + crate::float(s).map_err(|e| format!("ValueError: {e}")) +} + +impl ClinicalFloor { + /// Reads the floors `og vcf csq-rank` wrote into the header. + pub fn read(lines: &[String]) -> Result { + let mut f = ClinicalFloor::default(); + for l in lines.iter().take_while(|l| l.starts_with("##")) { + if !l.starts_with("##PGP_CALL_FLOOR=") { + continue; + } + floor_values(l, true, &mut f.call); + if let Some((lo, Some(hi))) = search(l, "HET_AB=", true) { + f.het_ab = (float(lo)?, float(hi)?); + } + if let Some((v, _)) = search(l, "HOM_AB=", false) { + f.hom_ab = float(v)?; + } + } + Ok(f) + } +} + +/// One ranked CSQ entry. +#[derive(Debug, Clone, PartialEq)] +pub struct CsqEntry { + pub impact: String, + pub gene: String, + pub consequence: String, +} + +/// `csq_rank`: the transcript class (MANE, CANONICAL or ANY), its entries, +/// and the frequency word (rare, common, unknown, or none when the CSQ has +/// no frequency field) from MAX_AF, else gnomADe_AF and gnomADg_AF, over +/// every entry. +pub fn csq_rank(fmt: &[String], csq: &str) -> (&'static str, Vec, &'static str) { + let ix: HashMap<&str, usize> = fmt + .iter() + .enumerate() + .map(|(i, k)| (k.as_str(), i)) + .collect(); + let entries: Vec> = csq + .split(',') + .filter(|t| !t.is_empty()) + .map(|t| t.split('|').collect::>()) + .filter(|v| v.len() == fmt.len()) + .collect(); + let get = |v: &[&str], k: &str| ix.get(k).map_or("", |&i| v[i]).to_owned(); + let mane: Vec<&Vec<&str>> = entries + .iter() + .filter(|v| !get(v, "MANE_SELECT").is_empty()) + .collect(); + let canon: Vec<&Vec<&str>> = entries + .iter() + .filter(|v| get(v, "CANONICAL") == "YES") + .collect(); + let (cls, chosen) = if !mane.is_empty() { + ("MANE", mane) + } else if !canon.is_empty() { + ("CANONICAL", canon) + } else { + ("ANY", entries.iter().collect()) + }; + let fields: Vec<&str> = if ix.contains_key("MAX_AF") { + vec!["MAX_AF"] + } else { + ["gnomADe_AF", "gnomADg_AF"] + .into_iter() + .filter(|f| ix.contains_key(f)) + .collect() + }; + let afs: Vec = entries + .iter() + .flat_map(|v| { + fields + .iter() + .flat_map(|f| { + get(v, f) + .split('&') + .filter_map(|a| pycompat::num::parse_float(a).ok()) + .collect::>() + }) + .collect::>() + }) + .collect(); + let freq = if fields.is_empty() { + "none" + } else { + match text::py_max(afs) { + None => "unknown", + Some(m) if m < COMMON_AF => "rare", + Some(_) => "common", + } + }; + let out = chosen + .iter() + .map(|v| CsqEntry { + impact: get(v, "IMPACT"), + gene: get(v, "SYMBOL"), + consequence: get(v, "Consequence").replace('&', ","), + }) + .collect(); + (cls, out, freq) +} + +/// The genotype check of one call: weak, the reasons, GQ, DP and AD. +pub fn call_check( + fmt: &str, + sample: &str, + floor: &ClinicalFloor, +) -> ( + bool, + Vec, + Option, + Option, + Option, +) { + let f = format_map(fmt, sample); + let gq = digits(f.get("GQ").copied()); + let dp = digits(f.get("DP").copied()); + let ad = f.get("AD").copied().unwrap_or(""); + let mut why = Vec::new(); + if let Some(g) = gq + && g < floor.call.gq + { + why.push(format!("GQ {g} < {}", floor.call.gq)); + } + if let Some(d) = dp + && d < floor.call.dp + { + why.push(format!("DP {d} < {}", floor.call.dp)); + } + let gt = f.get("GT").copied().unwrap_or("").replace('|', "/"); + let al: Vec<&str> = gt.split('/').collect(); + let depths: Vec<&str> = ad.split(',').collect(); + if al.len() == 2 + && al.iter().all(|a| isdigit(a)) + && depths.len() > 1 + && depths.iter().all(|x| isdigit(x)) + { + let n: Vec = depths.iter().filter_map(|x| crate::int(x).ok()).collect(); + let tot: i128 = n.iter().sum(); + if tot != 0 { + let ab = n[1..].iter().sum::() as f64 / tot as f64; + let (lo, hi) = floor.het_ab; + if (al[0] == "0") != (al[1] == "0") && !(lo <= ab && ab <= hi) { + why.push(format!( + "alt reads {} of the depth on a heterozygote", + pycompat::fmt::fixed(ab, 2) + )); + } else if al[0] == al[1] && al[1] != "0" && ab < floor.hom_ab { + why.push(format!( + "alt reads {} of the depth on a homozygous call", + pycompat::fmt::fixed(ab, 2) + )); + } + } + } + ( + !why.is_empty(), + why, + gq, + dp, + (!ad.is_empty()).then(|| ad.to_owned()), + ) +} + +/// One record of the clinical VCF with a HIGH-impact ranked entry. +#[derive(Debug, Clone, PartialEq)] +pub struct HighRecord { + pub genotype: String, + pub ps: Option, + /// Gene to its first HIGH consequence. + pub genes: IndexMap, + pub transcript: &'static str, + pub frequency: &'static str, + pub weak: bool, + pub why: Vec, + pub gq: Option, + pub dp: Option, + pub ad: Option, +} + +/// The clinical VCF read once: its records count and its HIGH-impact +/// records. An error is the note of a file that cannot be read. +#[derive(Debug, Clone)] +pub struct ClinicalVcf { + pub records: usize, + /// Keyed by `(chrom, pos, ref, alt)`; an error is the collector's note + /// (a POS that is no integer on a HIGH-impact record, a bad floor). + pub high: Result, String>, +} + +impl ClinicalVcf { + pub fn read(path: &str) -> Result { + let lines = text::lines(path).map_err(|e| e.unreadable(path))?; + let records = lines.iter().filter(|l| !l.starts_with('#')).count(); + Ok(ClinicalVcf { + records, + high: high_impact(&lines), + }) + } +} + +/// `csq_high_impact_genes`. +fn high_impact(lines: &[String]) -> Result, String> { + let floor = ClinicalFloor::read(lines)?; + let mut fmt: Option> = None; + let mut out = IndexMap::new(); + for line in lines { + if line.starts_with("##INFO= = line.split('\t').map(str::to_owned).collect(); + if r.len() < 8 { + continue; + } + let (cls, entries, freq) = csq_rank(fm, info_map(&r[7]).get("CSQ").copied().unwrap_or("")); + let mut genes: IndexMap = IndexMap::new(); + for e in entries.into_iter().filter(|e| e.impact == "HIGH") { + genes.entry(e.gene).or_insert(e.consequence); + } + if genes.is_empty() { + continue; + } + let gt = text::gt_of(&r).to_owned(); + let (weak, why, gq, dp, ad) = if r.len() > 9 { + call_check(&r[8], &r[9], &floor) + } else { + (false, Vec::new(), None, None, None) + }; + let pos = text::pos(&r)?; + out.insert( + (r[0].clone(), pos, r[3].clone(), r[4].clone()), + HighRecord { + genotype: gt, + ps: gt_field(&r, "PS"), + genes, + transcript: cls, + frequency: freq, + weak, + why, + gq, + dp, + ad, + }, + ); + } + Ok(out) +} + +/// A source of a candidate variant. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Kind { + ClinVar, + High, + Sv, +} + +impl Kind { + fn source(self) -> &'static str { + match self { + Kind::ClinVar => "ClinVar", + Kind::High => "HIGH impact", + Kind::Sv => "AnnotSV", + } + } +} + +/// A variant key the verdicts are counted by: a small variant, an SV by +/// its span, or a gene's called deletion. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +enum AKey { + Small(Key), + Sv(String, String, String, String), + Deletion(String), +} + +/// One candidate: the fields it is printed with (in their order), its key, +/// copies and ploidy, and its call's GT and phase set. +#[derive(Debug, Clone)] +struct Cand { + kind: Kind, + gene: String, + fields: IndexMap, + key: AKey, + copies: usize, + ploidy: usize, + gt: String, + ps: Option, +} + +/// `sf_verdicts`: for each distinct variant of a gene, `None` (reported) or +/// the reason it is not, and the partial-deficiency note of a reported gene +/// with a hypomorph allele. +fn sf_verdicts( + gene: &str, + rule: &Rule, + alleles: &IndexMap, +) -> (HashMap>, Option) { + let sev = |k: &AKey| match k { + AKey::Small(k) => rule.severity.get(k), + _ => None, + }; + let mut out: IndexMap> = IndexMap::new(); + let mut kept: IndexMap = IndexMap::new(); + for (k, cp) in alleles { + match sev(k) { + Some(a) if a.kind.as_deref() == Some("pseudodeficiency") => { + out.insert( + k.clone(), + Some(format!( + "{}: a pseudodeficiency allele, not counted", + a.name + )), + ); + } + _ => { + kept.insert(k.clone(), *cp); + } + } + } + let n: usize = kept.values().map(|(c, _)| c).sum(); + let ri = rule.report_if.as_str(); + let hemi = kept.values().any(|(_, p)| *p == 1); + for (k, (c, p)) in &kept { + let good = match ri { + "any" | "het_female_too" => true, + "biallelic" => n >= 2, + "hemizygous" => hemi || n >= 2, + _ => matches!(k, AKey::Small(s) if rule.variants.contains_key(s)) && *c >= 2 && *p == 2, + }; + let why = if good { + None + } else if ri == "hom_variant" { + Some(format!("{gene} is reported only for: {}", rule.table_text)) + } else if ri == "hemizygous" { + Some(format!( + "one heterozygous variant; {gene} is reported for {}", + rule.table_text + )) + } else { + Some(format!( + "one heterozygous variant; {gene} is reported with {}", + rule.table_text + )) + }; + out.insert(k.clone(), why); + } + let hypo: Vec<&str> = out + .iter() + .filter(|(_, why)| why.is_none()) + .filter_map(|(k, _)| sev(k).filter(|a| a.kind.as_deref() == Some("hypomorph"))) + .map(|a| a.name.as_str()) + .collect(); + let partial = (!hypo.is_empty()).then(|| { + format!( + "partial, not profound: {} is a hypomorph allele, so the deficiency it causes is partial", + hypo.join("; ") + ) + }); + (out.into_iter().collect(), partial) +} + +/// `sf_structural`: step 05's class 4 and 5 SVs in genes whose rule allows +/// them, and the class 4 or 5 deletions over a gene's sv_regions. +fn sf_structural( + inp: &DeletionInputs, + rules: &IndexMap, +) -> (Vec, bool, Option) { + let rows = match &inp.annotsv { + Err(e) => { + return ( + Vec::new(), + false, + Some(format!("the AnnotSV table could not be read: {e}")), + ); + } + Ok(None) => return (Vec::new(), false, None), + Ok(Some(rows)) => rows, + }; + let mut out = Vec::new(); + for x in rows { + if x.common || !matches!(x.class(), "4" | "5") { + continue; + } + let (chrom, span) = x.sv.split_once(':').unwrap_or((&x.sv, "")); + let (a, b) = span.split_once('-').unwrap_or((span, "")); + let chrom = if chrom.starts_with("chr") { + chrom.to_owned() + } else { + format!("chr{chrom}") + }; + let mut hits: IndexMap> = IndexMap::new(); + for g in &x.genes { + if rules.get(g).is_some_and(|r| r.sv_cn) { + hits.insert(g.clone(), None); + } + } + let sv_type = x.sv_type.as_deref().unwrap_or("None"); + let (an, bn) = ( + isdigit(a).then(|| crate::int(a).ok()).flatten(), + isdigit(b).then(|| crate::int(b).ok()).flatten(), + ); + for (g, rule) in rules { + for (c, rs, re, name) in &rule.sv_regions { + if let (Some(an), Some(bn)) = (an, bn) + && sv_type == "DEL" + && *c == chrom + && an <= *re + && bn >= *rs + && !hits.contains_key(g) + { + hits.insert(g.clone(), Some(name.clone())); + } + } + } + hits.sort_keys(); + for (g, via) in hits { + let mut f = IndexMap::new(); + f.insert("gene".into(), Value::Str(g.clone())); + f.insert( + "variant".into(), + Value::Str(format!("{sv_type} {chrom}:{a}-{b}")), + ); + f.insert("acmg_class".into(), Value::Str(x.class().to_owned())); + f.insert("caller".into(), opt_s(x.caller.as_deref())); + f.insert("support".into(), opt_int(x.support)); + f.insert("via".into(), opt_s(via.as_deref())); + out.push(Cand { + kind: Kind::Sv, + gene: g, + fields: f, + key: AKey::Sv( + chrom.clone(), + a.to_owned(), + b.to_owned(), + sv_type.to_owned(), + ), + copies: 1, + ploidy: 2, + gt: String::new(), + ps: None, + }); + } + } + (out, true, None) +} + +fn zyg_map(gt: &str) -> String { + match gt { + "0/1" | "1/0" | "0|1" | "1|0" => "het".into(), + "1/1" | "1|1" => "hom".into(), + _ => gt.into(), + } +} + +/// What the ACMG tier reads. +pub struct Inputs<'a> { + pub assets: &'a str, + /// Step 06's floor and hits, or the note of a hits VCF that cannot be + /// read; `None` without one. + pub hits: Option<&'a Result>, + /// The clinical VCF, or its note; `None` without one. + pub clinical: Option<&'a Result>, + pub deletions: &'a DeletionInputs, +} + +/// `acmg_sf_or_note`: the tier, or `Ok` of the unavailable form when a +/// list or an input cannot be read. An `Err` is the collector's note of the +/// whole clinical section. +pub fn acmg_sf(i: &Inputs) -> Result { + match tier(i)? { + Ok(v) => Ok(v), + Err(why) => Ok(obj([ + ("version", Value::Str(ACMG_SF_VERSION.into())), + ("unavailable", Value::Str(why)), + ("clinvar_hits", Value::Array(Vec::new())), + ("high_impact", Value::Array(Vec::new())), + ("structural", Value::Array(Vec::new())), + ("not_reported", Value::Array(Vec::new())), + ])), + } +} + +/// `acmg_sf_tier`: the outer error is the section's note, the inner one +/// the reason the tier is unavailable. +fn tier(i: &Inputs) -> Result, String> { + let rules = match acmg_sf_rules(i.assets) { + Ok(r) => r, + Err(e) => return Ok(Err(e)), + }; + let mut clinvar: Vec = Vec::new(); + if let Some(h) = i.hits { + let (floor, recs) = match h { + Ok(x) => x, + Err(e) => return Ok(Err(e.clone())), + }; + for r in recs.iter().filter(|r| r.len() >= 8) { + let row = Hit::parse(r, *floor)?; + let gt = text::gt_of(r).to_owned(); + let (copies, ploidy) = allele_copies(>); + for g in row.gene.split(',') { + if !rules.contains_key(g) { + continue; + } + let mut f = IndexMap::new(); + f.insert("gene".into(), Value::Str(g.to_owned())); + f.insert("variant".into(), Value::Str(row.variant())); + f.insert("genotype".into(), Value::Str(row.genotype.clone())); + f.insert("significance".into(), Value::Str(row.significance.clone())); + f.insert("stars".into(), Value::Int(row.stars.into())); + f.insert("low_confidence".into(), Value::Bool(row.low_confidence)); + clinvar.push(Cand { + kind: Kind::ClinVar, + gene: g.to_owned(), + fields: f, + key: AKey::Small(row.key()), + copies, + ploidy, + gt: gt.clone(), + ps: gt_field(r, "PS"), + }); + } + } + } + let mut high: Vec = Vec::new(); + if let Some(c) = i.clinical { + let c = match c { + Ok(c) => c, + Err(e) => return Ok(Err(e.clone())), + }; + let mut recs: Vec<(&Key, &HighRecord)> = + c.high.as_ref().map_err(Clone::clone)?.iter().collect(); + recs.sort_by(|a, b| a.0.cmp(b.0)); + for ((ch, pos, r, a), x) in recs { + if x.frequency == "common" { + continue; + } + let mut genes: Vec<(&String, &String)> = x.genes.iter().collect(); + genes.sort(); + for (g, cons) in genes { + if !rules.contains_key(g) { + continue; + } + let (copies, ploidy) = allele_copies(&x.genotype); + let mut f = IndexMap::new(); + f.insert("gene".into(), Value::Str(g.clone())); + f.insert("variant".into(), Value::Str(format!("{ch}:{pos} {r}>{a}"))); + f.insert("genotype".into(), Value::Str(zyg_map(&x.genotype))); + f.insert("consequence".into(), Value::Str(cons.clone())); + f.insert("transcript".into(), Value::Str(x.transcript.into())); + f.insert("frequency".into(), Value::Str(x.frequency.into())); + f.insert( + "frequency_unknown".into(), + Value::Bool(x.frequency == "unknown"), + ); + f.insert("weak_call".into(), Value::Bool(x.weak)); + f.insert( + "call_note".into(), + if x.why.is_empty() { + Value::Null + } else { + Value::Str(x.why.join("; ")) + }, + ); + f.insert("gq".into(), opt_int(x.gq)); + f.insert("dp".into(), opt_int(x.dp)); + f.insert("ad".into(), opt_s(x.ad.as_deref())); + high.push(Cand { + kind: Kind::High, + gene: g.clone(), + fields: f, + key: AKey::Small((ch.clone(), *pos, r.clone(), a.clone())), + copies, + ploidy, + gt: x.genotype.clone(), + ps: x.ps.clone(), + }); + } + } + } + let (svs, sv_checked, sv_note) = sf_structural(i.deletions, &rules); + + // One verdict per gene, over its distinct variants. + let mut alleles: IndexMap> = IndexMap::new(); + for x in clinvar.iter().chain(&high) { + let p = if x.ploidy == 0 { 2 } else { x.ploidy }; + alleles + .entry(x.gene.clone()) + .or_default() + .entry(x.key.clone()) + .or_insert((x.copies.max(1), p)); + } + for x in &svs { + alleles + .entry(x.gene.clone()) + .or_default() + .entry(x.key.clone()) + .or_insert((1, 2)); + } + let (spans, two_hit_note): (Spans, Option) = match carrier_genes(i.assets) { + Ok(g) => (g.into_iter().map(|(k, v)| (k, v.span)).collect(), None), + Err(e) => ( + IndexMap::new(), + Some(format!( + "A deletion as the second hit: only AnnotSV's gene names were read; step 35's \ + deletions need the gene spans of {e}" + )), + ), + }; + let (dels, _, _) = gene_deletions(i.deletions, &spans); + let mut seen = HashSet::new(); + let mut variants = Vec::new(); + for x in clinvar.iter().chain(&high) { + let AKey::Small(k) = &x.key else { continue }; + let pseudo = rules[&x.gene].severity_kind(k) == Some("pseudodeficiency"); + if !pseudo && seen.insert((x.gene.clone(), k.clone())) { + variants.push(Variant { + gene: x.gene.clone(), + key: k.clone(), + chrom: k.0.clone(), + pos: k.1, + text: match &x.fields["variant"] { + Value::Str(s) => s.clone(), + _ => String::new(), + }, + genotype: match &x.fields["genotype"] { + Value::Str(s) => s.clone(), + _ => String::new(), + }, + copies: x.copies, + ploidy: if x.ploidy == 0 { 2 } else { x.ploidy }, + gt: x.gt.clone(), + ps: x.ps.clone(), + allele: None, + }); + } + } + let found = two_hits(&variants, &dels); + let mut two: HashMap<(String, Key), String> = HashMap::new(); + for g in found.entries.keys() { + if matches!(rules[g].report_if.as_str(), "biallelic" | "hemizygous") { + if let Some(a) = alleles.get_mut(g) { + a.insert(AKey::Deletion(g.clone()), (1, 2)); + } + two.extend( + found + .pairs + .iter() + .filter(|(k, _)| &k.0 == g) + .map(|(k, t)| (k.clone(), t.clone())), + ); + } + } + let verdict: HashMap<&String, _> = alleles + .iter() + .map(|(g, a)| (g, sf_verdicts(g, &rules[g], a))) + .collect(); + let mut reported: [Vec; 3] = Default::default(); + let mut not_reported = Vec::new(); + for x in clinvar.into_iter().chain(high).chain(svs) { + let (whys, partial) = &verdict[&x.gene]; + let why = whys.get(&x.key).cloned().flatten(); + let rule = &rules[&x.gene]; + let mut f = x.fields; + if let AKey::Small(k) = &x.key + && found.hemi.contains_key(&(x.gene.clone(), k.clone())) + { + f.insert("genotype".into(), Value::Str("hemizygous".into())); + } + let slot = match x.kind { + Kind::ClinVar => 0, + Kind::High => 1, + Kind::Sv => 2, + }; + match why { + None => { + f.insert("rule".into(), Value::Str(rule.table_text.clone())); + f.insert("rule_note".into(), opt_s(rule.note.as_deref())); + f.insert("severity".into(), opt_s(partial.as_deref())); + if x.kind != Kind::Sv + && let AKey::Small(k) = &x.key + && let Some(t) = two.get(&(x.gene.clone(), k.clone())) + { + f.insert("two_hit".into(), Value::Str(t.clone())); + } + reported[slot].push(Value::Object(f)); + } + Some(reason) => not_reported.push(obj([ + ("gene", Value::Str(x.gene.clone())), + ("variant", f["variant"].clone()), + ( + "genotype", + f.get("genotype").cloned().unwrap_or(Value::Null), + ), + ("source", Value::Str(x.kind.source().into())), + ("reason", Value::Str(reason)), + ])), + } + } + let [clinvar_rows, high_rows, sv_rows] = reported; + let flag = + |x: &Value, k: &str| matches!(x, Value::Object(m) if m.get(k) == Some(&Value::Bool(true))); + let sure = high_rows + .iter() + .filter(|x| !(flag(x, "weak_call") || flag(x, "frequency_unknown"))) + .count(); + let weak = high_rows.iter().filter(|x| flag(x, "weak_call")).count(); + let unknown = high_rows + .iter() + .filter(|x| flag(x, "frequency_unknown")) + .count(); + Ok(Ok(obj([ + ("version", Value::Str(ACMG_SF_VERSION.into())), + ("genes_on_list", Value::Int(rules.len() as i128)), + ("rules", Value::Str("assets/acmg_sf.tsv".into())), + ("clinvar_hits", Value::Array(clinvar_rows)), + ("clinvar_checked", Value::Bool(i.hits.is_some())), + ("high_impact", Value::Array(high_rows)), + ("high_impact_checked", Value::Bool(i.clinical.is_some())), + ("high_impact_rare", Value::Int(sure as i128)), + ("high_impact_weak", Value::Int(weak as i128)), + ("high_impact_frequency_unknown", Value::Int(unknown as i128)), + ("structural", Value::Array(sv_rows)), + ("structural_checked", Value::Bool(sv_checked)), + ("structural_note", opt_s(sv_note.as_deref())), + ("not_reported", Value::Array(not_reported)), + ("two_hit_note", opt_s(two_hit_note.as_deref())), + ]))) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fmt(s: &str) -> Vec { + s.split('|').map(str::to_owned).collect() + } + + #[test] + fn mane_then_canonical_then_any() { + let f = fmt("Allele|IMPACT|SYMBOL|Consequence|CANONICAL|MANE_SELECT|gnomADe_AF"); + let (c, e, q) = csq_rank( + &f, + "T|HIGH|A|stop_gained|YES||0.5,T|LOW|A|synonymous_variant||NM_1|0.001", + ); + assert_eq!((c, q), ("MANE", "common")); + assert_eq!(e.len(), 1); + assert_eq!(e[0].impact, "LOW"); + let (c, _, q) = csq_rank(&f, "T|HIGH|A|stop_gained&splice|YES||"); + assert_eq!((c, q), ("CANONICAL", "unknown")); + let f2 = fmt("Allele|IMPACT|SYMBOL|Consequence"); + let (c, e, q) = csq_rank(&f2, "T|HIGH|A|a&b"); + assert_eq!((c, q), ("ANY", "none")); + assert_eq!(e[0].consequence, "a,b"); + } + + #[test] + fn the_genotype_check() { + let f = ClinicalFloor::default(); + let (weak, why, gq, dp, ad) = call_check("GT:GQ:DP:AD", "0/1:20:30:25,5", &f); + assert!(weak); + assert_eq!( + why, + [ + "GQ 20 < 30", + "alt reads 0.17 of the depth on a heterozygote" + ] + ); + assert_eq!((gq, dp, ad.as_deref()), (Some(20), Some(30), Some("25,5"))); + let (weak, why, ..) = call_check("GT:AD", "1|1:5,15", &f); + assert!(weak); + assert_eq!(why, ["alt reads 0.75 of the depth on a homozygous call"]); + let (weak, ..) = call_check("GT:GQ:DP:AD", "0/1:40:30:15,15", &f); + assert!(!weak); + } + + #[test] + fn the_floors_of_the_header() { + let l = |s: &str| vec![s.to_owned(), "#CHROM".to_owned()]; + let f = ClinicalFloor::read(&l( + "##PGP_CALL_FLOOR=GQ=20,DP=5,HET_AB=0.25-0.75,HOM_AB=0.9", + )) + .unwrap(); + assert_eq!(f.call, Floor { gq: 20, dp: 5 }); + assert_eq!((f.het_ab, f.hom_ab), ((0.25, 0.75), 0.9)); + assert!(ClinicalFloor::read(&l("##PGP_CALL_FLOOR=HOM_AB=0..9")).is_err()); + } +} diff --git a/rust/crates/og-summary/src/collect/assets.rs b/rust/crates/og-summary/src/collect/assets.rs new file mode 100644 index 000000000..a71eeb613 --- /dev/null +++ b/rust/crates/og-summary/src/collect/assets.rs @@ -0,0 +1,297 @@ +//! The two gene lists the clinical sections read from `assets/`: +//! `acmg_sf.tsv` (the ACMG SF v3.3 genes and the reporting rule of each, +//! Lee et al., Genet Med 2025, PMID 40568962) and `carrier_genes.tsv` (the +//! genes of the carrier list and their spans). +//! +//! A missing or broken list is an error with the words the reports print; +//! the section that needs it says the list is unavailable and keeps its +//! other values. + +use crate::{basename, join}; +use indexmap::IndexMap; +use pycompat::csv::{DictReader, DictRow, Quoting}; + +/// The list version both reports print. +pub const ACMG_SF_VERSION: &str = "ACMG SF v3.3"; +/// The rules a gene of the ACMG list can have. +pub const SF_REPORT_IF: [&str; 5] = [ + "any", + "biallelic", + "hom_variant", + "hemizygous", + "het_female_too", +]; +/// The kinds of a severity entry. +pub const SEVERITY_KINDS: [&str; 2] = ["hypomorph", "pseudodeficiency"]; + +/// A variant as `(chrom, pos, ref, alt)`. +pub type Key = (String, i128, String, String); + +/// What a severity entry or a variants entry names. +#[derive(Debug, Clone, PartialEq)] +pub struct Allele { + /// `hypomorph` or `pseudodeficiency`; `None` in a variants cell. + pub kind: Option, + pub name: String, +} + +/// The reporting rule of one ACMG gene. +#[derive(Debug, Clone)] +pub struct Rule { + pub report_if: String, + pub variants: IndexMap, + /// AnnotSV's class 4 and 5 SVs count in this gene. + pub sv_cn: bool, + /// Regions where a deletion counts for the gene: chrom, start, end, name. + pub sv_regions: Vec<(String, i128, i128, String)>, + pub severity: IndexMap, + pub table_text: String, + pub note: Option, +} + +impl Rule { + pub fn severity_kind(&self, key: &Key) -> Option<&str> { + self.severity.get(key).and_then(|a| a.kind.as_deref()) + } +} + +/// One gene of the carrier list. +#[derive(Debug, Clone)] +pub struct CarrierGene { + pub inheritance: String, + pub condition: String, + pub severity: IndexMap, + /// GRCh38, 1-based, inclusive. + pub span: (String, i128, i128), +} + +/// The rows of `assets/` without its `#` lines, cells as written +/// (no quoting). +pub fn asset_rows(assets: &str, name: &str) -> Result, String> { + let p = join(assets, name); + let missing = || { + format!( + "{name} is not in {assets}: the report needs the repository's assets/ folder \ + (render_report.py --assets)" + ) + }; + if !crate::is_file(&p) { + return Err(missing()); + } + let bytes = std::fs::read(&p).map_err(|e| crate::os_error(&e, &p))?; + let text = String::from_utf8(bytes).map_err(|e| { + format!( + "UnicodeDecodeError: 'utf-8' codec can't decode {}: {e}", + basename(&p) + ) + })?; + let lines = text + .split_inclusive('\n') + .filter(|l| !l.starts_with('#')) + .map(|l| Ok::<_, std::io::Error>(l.to_owned())); + DictReader::new(lines, '\t', Quoting::None) + .collect::, _>>() + .map_err(|e| format!("{}: {e}", e.exception())) +} + +/// `row[key]`, its error as `str(KeyError)`. +fn col<'a>(r: &'a DictRow, key: &str) -> Result, String> { + r.get(key).ok_or_else(|| crate::repr(key)) +} + +fn int(s: Option<&str>) -> Result { + match s { + None => Err( + "int() argument must be a string, a bytes-like object or a real number, not 'NoneType'" + .into(), + ), + Some(s) => crate::int(s), + } +} + +/// `'chr6:26092913:G:A'` as a key. +pub fn allele_key(locus: &str) -> Result { + let f: Vec<&str> = locus.split(':').collect(); + match f.as_slice() { + [c, pos, r, a] => Ok(( + (*c).to_owned(), + crate::int(pos)?, + (*r).to_owned(), + (*a).to_owned(), + )), + _ if f.len() < 4 => Err(format!( + "not enough values to unpack (expected 4, got {})", + f.len() + )), + _ => Err(format!( + "too many values to unpack (expected 4, got {})", + f.len() + )), + } +} + +/// The variants or severity cell: `.` or entries separated by `;`, each +/// `[kind ]chrom:pos:ref:alt name`. +pub fn parse_alleles(text: Option<&str>, kinds: bool) -> Result, String> { + let mut out = IndexMap::new(); + for e in text.filter(|t| !t.is_empty()).unwrap_or(".").split(';') { + let e = pycompat::text::strip(e, None); + if e.is_empty() || e == "." { + continue; + } + let mut parts: Vec<&str> = e.splitn(if kinds { 3 } else { 2 }, ' ').collect(); + if kinds && parts.len() < 2 { + return Err(format!( + "severity entry needs a kind and a chrom:pos:ref:alt: {e}" + )); + } + let kind = if kinds { + let k = parts.remove(0); + if !SEVERITY_KINDS.contains(&k) { + return Err(format!( + "severity '{k}' is not one of {}: {e}", + SEVERITY_KINDS.join(", ") + )); + } + Some(k.to_owned()) + } else { + None + }; + let name = parts.get(1).copied().unwrap_or(parts[0]).to_owned(); + out.insert(allele_key(parts[0])?, Allele { kind, name }); + } + Ok(out) +} + +/// `{gene: rule}` of `acmg_sf.tsv`, in the file's order. +pub fn acmg_sf_rules(assets: &str) -> Result, String> { + let mut rules: IndexMap = IndexMap::new(); + for (n, r) in asset_rows(assets, "acmg_sf.tsv")?.iter().enumerate() { + let one = || -> Result<(String, Rule), String> { + let g = col(r, "gene")?.unwrap_or("None").to_owned(); + let report_if = col(r, "report_if")?; + let sv_cn = col(r, "sv_cn")?; + let ri_ok = report_if.is_some_and(|x| SF_REPORT_IF.contains(&x)); + if !ri_ok || !matches!(sv_cn, Some("yes" | "no")) { + return Err(format!( + "report_if must be one of {}, sv_cn yes or no", + SF_REPORT_IF.join(", ") + )); + } + let report_if = report_if.unwrap_or_default(); + if sv_cn == Some("yes") && !matches!(report_if, "any" | "het_female_too") { + return Err( + "sv_cn yes needs a rule that one variant meets (any, het_female_too)".into(), + ); + } + let mut regions = Vec::new(); + for e in col(r, "sv_regions")? + .filter(|x| !x.is_empty()) + .unwrap_or(".") + .split(';') + { + let e = pycompat::text::strip(e, None); + if e.is_empty() || e == "." { + continue; + } + let (loc, name) = e.split_once(' ').unwrap_or((e, "")); + let (c, span) = loc.split_once(':').unwrap_or((loc, "")); + let (a, b) = span.split_once('-').unwrap_or((span, "")); + regions.push(( + c.to_owned(), + crate::int(a)?, + crate::int(b)?, + if name.is_empty() { loc } else { name }.to_owned(), + )); + } + let variants = parse_alleles(col(r, "variants")?, false)?; + if report_if == "hom_variant" && variants.is_empty() { + return Err("report_if hom_variant needs its variants".into()); + } + if rules.contains_key(&g) { + return Err("the gene is on two rows".into()); + } + let inheritance = col(r, "inheritance")?; + let _ = inheritance; + let severity = parse_alleles(col(r, "severity")?, true)?; + let table_text = col(r, "table_text")?.unwrap_or("None").to_owned(); + let note = col(r, "note")? + .filter(|x| !matches!(*x, "" | ".")) + .map(str::to_owned); + Ok(( + g, + Rule { + report_if: report_if.to_owned(), + variants, + sv_cn: sv_cn == Some("yes"), + sv_regions: regions, + severity, + table_text, + note, + }, + )) + }; + let (g, rule) = one().map_err(|e| format!("acmg_sf.tsv row {}: {e}", n + 1))?; + rules.insert(g, rule); + } + if rules.is_empty() { + return Err("acmg_sf.tsv has no gene".into()); + } + Ok(rules) +} + +/// `{gene: row}` of `carrier_genes.tsv`; a gene on two rows keeps its +/// first place and its last row. +pub fn carrier_genes(assets: &str) -> Result, String> { + let mut out = IndexMap::new(); + for (n, r) in asset_rows(assets, "carrier_genes.tsv")?.iter().enumerate() { + let one = || -> Result<(String, CarrierGene), String> { + let g = col(r, "gene")?.unwrap_or("None").to_owned(); + let inheritance = col(r, "inheritance")?.unwrap_or("None").to_owned(); + col(r, "lists")?; + let condition = col(r, "condition")?.unwrap_or("None").to_owned(); + let severity = parse_alleles(r.get("severity").flatten(), true)?; + let chrom = col(r, "chrom")?.unwrap_or("None").to_owned(); + let span = (chrom, int(col(r, "start")?)?, int(col(r, "end")?)?); + Ok(( + g, + CarrierGene { + inheritance, + condition, + severity, + span, + }, + )) + }; + let (g, x) = one().map_err(|e| format!("carrier_genes.tsv row {}: {e}", n + 1))?; + out.insert(g, x); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn alleles_read_with_and_without_kinds() { + let v = parse_alleles(Some("chr6:10:G:A C282Y; chr6:20:C:G"), false).unwrap(); + assert_eq!(v.len(), 2); + assert_eq!( + v[&("chr6".into(), 10, "G".into(), "A".into())].name, + "C282Y" + ); + assert_eq!( + v[&("chr6".into(), 20, "C".into(), "G".into())].name, + "chr6:20:C:G" + ); + let s = parse_alleles(Some("hypomorph chr1:5:A:T p.X"), true).unwrap(); + assert_eq!( + s.values().next().unwrap().kind.as_deref(), + Some("hypomorph") + ); + assert!(parse_alleles(Some("odd chr1:5:A:T"), true).is_err()); + assert!(parse_alleles(Some("chr1:5:A"), false).is_err()); + assert!(parse_alleles(None, true).unwrap().is_empty()); + } +} diff --git a/rust/crates/og-summary/src/collect/carrier.rs b/rust/crates/og-summary/src/collect/carrier.rs new file mode 100644 index 000000000..852e35bf3 --- /dev/null +++ b/rust/crates/og-summary/src/collect/carrier.rs @@ -0,0 +1,146 @@ +//! The sample's own carrier list: step 06's ClinVar hits in the genes of +//! `assets/carrier_genes.tsv`, one person, no partner needed. An allele the +//! list marks hypomorph or pseudodeficiency is set aside, never a carrier +//! finding on its own. The genes of autosomal recessive conditions that a +//! called deletion and a hit both reach are listed as two hits, and a hit +//! called homozygous inside a deletion is shown as hemizygous. + +use super::assets::carrier_genes; +use super::clinvar::{Floor, Hit, HitRecords, allele_copies, carrier_genotype, obj}; +use super::text::{self, gt_field}; +use super::two_hits::{DeletionInputs, Spans, Variant, gene_deletions, two_hits}; +use indexmap::IndexMap; +use pycompat::json::Value; + +/// The section's data: the list, or why it could not be made. +#[derive(Debug, Clone, PartialEq)] +pub struct Carrier(pub Value); + +impl Carrier { + /// `carrier_list_or_note`. An `Err` is the collector's note (a hit + /// whose POS is no integer). + pub fn read( + assets: &str, + hits: &Result, + deletions: &DeletionInputs, + ) -> Result { + let unavailable = |e: String| Carrier(obj([("unavailable", Value::Str(e))])); + let genes = match carrier_genes(assets) { + Ok(g) => g, + Err(e) => return Ok(unavailable(e)), + }; + let recs = match hits { + Ok((_, r)) => r, + Err(e) => return Ok(unavailable(e.clone())), + }; + let mut items: Vec> = Vec::new(); + // Which list each item is on: carriers (false) or set aside (true). + let mut aside: Vec = Vec::new(); + let mut variants: Vec<(Variant, usize)> = Vec::new(); + for r in recs.iter().filter(|r| r.len() >= 8) { + let row = Hit::parse(r, Floor::default())?; + let gt = text::gt_of(r).to_owned(); + let (copies, ploidy) = allele_copies(>); + let key = row.key(); + for g in row.gene.split(',') { + let Some(x) = genes.get(g) else { continue }; + let mut item = IndexMap::new(); + item.insert("gene".into(), Value::Str(g.to_owned())); + item.insert("variant".into(), Value::Str(row.variant())); + item.insert("genotype".into(), Value::Str(carrier_genotype(>, &row))); + item.insert("significance".into(), Value::Str(row.significance.clone())); + item.insert("stars".into(), Value::Int(row.stars.into())); + item.insert("inheritance".into(), Value::Str(x.inheritance.clone())); + item.insert("condition".into(), Value::Str(x.condition.clone())); + let sev = x.severity.get(&key); + if let Some(s) = sev { + item.insert( + "kind".into(), + s.kind.clone().map_or(Value::Null, Value::Str), + ); + item.insert("name".into(), Value::Str(s.name.clone())); + } + let genotype = match &item["genotype"] { + Value::Str(s) => s.clone(), + _ => String::new(), + }; + items.push(item); + aside.push(sev.is_some()); + if sev.is_none_or(|s| s.kind.as_deref() == Some("hypomorph")) { + variants.push(( + Variant { + gene: g.to_owned(), + key: key.clone(), + chrom: row.chrom.clone(), + pos: row.pos, + text: row.variant(), + genotype, + copies, + ploidy, + gt: gt.clone(), + ps: gt_field(r, "PS"), + allele: sev.map(|s| s.name.clone()), + }, + items.len() - 1, + )); + } + } + } + let spans: Spans = genes + .iter() + .map(|(g, x)| (g.clone(), x.span.clone())) + .collect(); + let (dels, read, unread) = gene_deletions(deletions, &spans); + let vs: Vec = variants.iter().map(|(v, _)| v.clone()).collect(); + let found = two_hits(&vs, &dels); + for (v, at) in &variants { + if let Some(t) = found.hemi.get(&(v.gene.clone(), v.key.clone())) { + let item = &mut items[*at]; + item.insert("genotype".into(), Value::Str("hemizygous".into())); + item.insert( + "hemizygous".into(), + Value::Str(format!("called homozygous inside {t}")), + ); + } + } + let mut rows: Vec<(&String, _)> = found.entries.iter().collect(); + rows.sort_by(|a, b| a.0.cmp(b.0)); + let two: Vec = rows + .into_iter() + .filter(|(g, _)| genes[*g].inheritance == "AR") + .map(|(g, e)| { + let mut m = e.to_json(); + m.insert("condition".into(), Value::Str(genes[g].condition.clone())); + m.insert("inheritance".into(), Value::Str("AR".into())); + Value::Object(m) + }) + .collect(); + let (mut carriers, mut set_aside) = (Vec::new(), Vec::new()); + for (item, a) in items.into_iter().zip(aside) { + if a { + set_aside.push(Value::Object(item)); + } else { + carriers.push(Value::Object(item)); + } + } + let strs = |v: Vec| Value::Array(v.into_iter().map(Value::Str).collect()); + Ok(Carrier(obj([ + ("genes_on_list", Value::Int(genes.len() as i128)), + ("list", Value::Str("assets/carrier_genes.tsv".into())), + ("carriers", Value::Array(carriers)), + ("set_aside", Value::Array(set_aside)), + ( + "two_hits", + obj([ + ("genes", Value::Array(two)), + ("deletions_read", strs(read)), + ("deletions_not_read", strs(unread)), + ]), + ), + ]))) + } + + pub fn to_json(&self) -> Value { + self.0.clone() + } +} diff --git a/rust/crates/og-summary/src/collect/clinical.rs b/rust/crates/og-summary/src/collect/clinical.rs new file mode 100644 index 000000000..6dc161381 --- /dev/null +++ b/rust/crates/og-summary/src/collect/clinical.rs @@ -0,0 +1,208 @@ +//! The clinical variant filter (step 23) with its ACMG secondary-findings +//! tier, and the variant prioritization (step 31, slivar) with its +//! compound-het pairs phased from the reads. +//! +//! The bash pipeline writes summary tables; the Nextflow report gets the +//! VCFs instead, so each section reads the table when there is one and +//! counts the VCF's records otherwise. + +use super::acmg::{self, ClinicalVcf}; +use super::clinvar::obj; +use super::text::{self, cell, digits}; +use indexmap::IndexMap; +use pycompat::csv::DictRow; +use pycompat::json::Value; +use std::collections::BTreeSet; + +/// The trans pairs the report lists by name; the rest are counted. +pub const COMPHET_TRANS_SHOWN: usize = 20; + +fn rows_or_note(path: &str) -> Result, String> { + text::rows(path).map_err(|e| e.note()) +} + +fn int(n: usize) -> Value { + Value::Int(n as i128) +} + +/// The data of the clinical section. +#[derive(Debug, Clone, PartialEq)] +pub struct Clinical(pub Value); + +impl Clinical { + /// `sec_clinical`: `table` is step 23's summary table, `vcf` the + /// clinical VCF read once (both full paths). + pub fn read( + table: Option<&str>, + vcf: Option<&Result>, + tier: &acmg::Inputs, + ) -> Result { + let Some(table) = table else { + // The Nextflow report: the record count only (one record per + // variant, as in the table). + let v = match vcf { + Some(Ok(v)) => v, + Some(Err(e)) => return Err(e.clone()), + None => return Ok(Clinical(Value::Null)), + }; + return Ok(Clinical(obj([ + ("variants", int(v.records)), + ("acmg_sf", acmg::acmg_sf(tier)?), + ]))); + }; + let rows = rows_or_note(table)?; + let mut by: IndexMap = IndexMap::new(); + for r in &rows { + let k = cell(r, "IMPACT").filter(|v| !v.is_empty()).unwrap_or("."); + *by.entry(k.to_owned()).or_default() += 1; + } + let genes: BTreeSet<&str> = rows + .iter() + .filter_map(|r| cell(r, "GENE")) + .filter(|g| !matches!(*g, "" | ".")) + .collect(); + let mut m: IndexMap = IndexMap::new(); + m.insert("variants".into(), int(rows.len())); + m.insert( + "by_impact".into(), + Value::Object(by.into_iter().map(|(k, v)| (k, int(v))).collect()), + ); + m.insert("genes".into(), int(genes.len())); + m.insert("acmg_sf".into(), acmg::acmg_sf(tier)?); + // Step 23's tier and call check columns; an older table has neither. + if let Some(r) = rows.first() { + if r.get("FREQ_TIER").is_some() { + let n = rows + .iter() + .filter(|r| cell(r, "FREQ_TIER") == Some("unknown")) + .count(); + m.insert("frequency_unknown".into(), int(n)); + } + if r.get("CALL_CHECK").is_some() { + let n = rows + .iter() + .filter(|r| cell(r, "CALL_CHECK") == Some("weak")) + .count(); + m.insert("weak_calls".into(), int(n)); + } + } + Ok(Clinical(Value::Object(m))) + } + + pub fn to_json(&self) -> Value { + self.0.clone() + } +} + +/// The data of the slivar section. +#[derive(Debug, Clone, PartialEq)] +pub struct Slivar(pub Value); + +impl Slivar { + /// `sec_slivar` with step 31's summary table (`table`) or, for the + /// Nextflow report, its prioritized VCF (`vcf`); full paths. + pub fn read(table: Option<&str>, vcf: Option<&str>, s: &str) -> Result { + let mut m: IndexMap = IndexMap::new(); + let Some(p) = table else { + let Some(v) = vcf else { + return Ok(Slivar(Value::Null)); + }; + let n = text::vcf_records(v).map_err(|e| e.unreadable(v))?.len(); + m.insert("prioritized".into(), int(n)); + comphet_phase( + &crate::join(text::dirname(v), &format!("{s}_compound_het_pairs.tsv")), + &mut m, + )?; + return Ok(Slivar(Value::Object(m))); + }; + let rows = rows_or_note(p)?; + m.insert("prioritized".into(), int(rows.len())); + // Step 31's frequency tier: prioritized for the compound-het search, not rare. + if rows.first().is_some_and(|r| r.get("FREQ_TIER").is_some()) { + let n = rows + .iter() + .filter(|r| cell(r, "FREQ_TIER") == Some("unknown")) + .count(); + m.insert("frequency_unknown".into(), int(n)); + } + let dir = text::dirname(p); + let ch = crate::join(dir, &format!("{s}_compound_hets.tsv")); + if crate::is_file(&ch) { + let c = rows_or_note(&ch)?; + let genes: std::collections::HashSet<&str> = c + .iter() + .filter_map(|r| cell(r, "GENE")) + .filter(|g| !g.is_empty()) + .collect(); + m.insert("compound_het_variants".into(), int(c.len())); + m.insert("compound_het_genes".into(), int(genes.len())); + } + comphet_phase( + &crate::join(dir, &format!("{s}_compound_het_pairs.tsv")), + &mut m, + )?; + Ok(Slivar(Value::Object(m))) + } + + pub fn to_json(&self) -> Value { + self.0.clone() + } +} + +/// Step 31's pairs table: the pairs by phase from the reads, and the trans +/// ones (both copies of the gene hit) by name. A run from before the table +/// adds none of these keys. +fn comphet_phase(path: &str, m: &mut IndexMap) -> Result<(), String> { + if !crate::is_file(path) { + return Ok(()); + } + let rows = rows_or_note(path)?; + let phase = obj(["trans", "cis", "unknown"].map(|k| { + ( + k, + int(rows.iter().filter(|r| cell(r, "PHASE") == Some(k)).count()), + ) + })); + let py = |r: &DictRow, k: &str| cell(r, k).unwrap_or("None").to_owned(); + let n = |r: &DictRow, k: &str| digits(cell(r, k)).unwrap_or(0); + let trans: Vec = rows + .iter() + .filter(|r| cell(r, "PHASE") == Some("trans")) + .take(COMPHET_TRANS_SHOWN) + .map(|r| { + let gene = match r.get("GENE") { + None => Value::Str(String::new()), + Some(v) => v.map_or(Value::Null, |g| Value::Str(g.to_owned())), + }; + obj([ + ("gene", gene), + ( + "variants", + Value::Str(format!( + "{}:{} {}>{} + {}:{} {}>{}", + py(r, "CHROM"), + py(r, "POS_1"), + py(r, "REF_1"), + py(r, "ALT_1"), + py(r, "CHROM"), + py(r, "POS_2"), + py(r, "REF_2"), + py(r, "ALT_2") + )), + ), + ( + "reads", + Value::Str(format!( + "{} of {}", + py(r, "TRANS_READS"), + n(r, "TRANS_READS") + n(r, "CIS_READS") + )), + ), + ]) + }) + .collect(); + m.insert("compound_het_pairs".into(), int(rows.len())); + m.insert("compound_het_phase".into(), phase); + m.insert("compound_het_trans".into(), Value::Array(trans)); + Ok(()) +} diff --git a/rust/crates/og-summary/src/collect/clinvar.rs b/rust/crates/og-summary/src/collect/clinvar.rs new file mode 100644 index 000000000..7350c5a6b --- /dev/null +++ b/rust/crates/og-summary/src/collect/clinvar.rs @@ -0,0 +1,799 @@ +//! The ClinVar screen (step 06): each P/LP hit with its review stars, its +//! call quality against step 06's floor, gnomAD AF and rsIDs from step 13's +//! VEP output, and the cDNA read check, the cDNA scan and the always-report +//! deep intronic sites when step 06 wrote them. +//! +//! The hits VCF is small and three sections read it (this one, the ACMG +//! tier of the clinical section and the carrier list), so it is read once +//! ([`HitsVcf`]) and each section parses the rows it needs. + +use super::text::{self, ReadError, digits, format_map, gt_of, info_map, py_max}; +use indexmap::IndexMap; +use og_clinvar::{ReviewStatus, combined_note, half_call, zygosity}; +use pycompat::json::Value; +use std::collections::HashMap; + +/// gnomAD AF above which a P/LP hit is marked common. +pub const COMMON_AF: f64 = 0.01; + +/// Step 06's call floor: a hit whose GQ or DP is below it is low confidence. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Floor { + pub gq: i128, + pub dp: i128, +} + +impl Default for Floor { + fn default() -> Self { + Floor { gq: 30, dp: 10 } + } +} + +impl Floor { + fn to_json(self) -> Value { + obj([("GQ", Value::Int(self.gq)), ("DP", Value::Int(self.dp))]) + } +} + +/// `(GQ|DP)=(\d+)` over a header line: every match, the last one wins. +pub fn floor_values(line: &str, word_start: bool, floor: &mut Floor) { + let b = line.as_bytes(); + for key in ["GQ=", "DP="] { + let mut from = 0; + while let Some(i) = line[from..].find(key).map(|i| i + from) { + from = i + 1; + if word_start && i > 0 && (b[i - 1].is_ascii_alphanumeric() || b[i - 1] == b'_') { + continue; + } + let rest = &line[i + 3..]; + let n = rest.bytes().take_while(u8::is_ascii_digit).count(); + if n > 0 + && let Ok(v) = crate::int(&rest[..n]) + { + // Python takes the matches in text order; with one key per + // pass, the last of each key is the one that stays. + if key == "GQ=" { + floor.gq = v; + } else { + floor.dp = v; + } + } + } + } +} + +/// A variant as `(chrom, pos, ref, alt)`. +pub type VarKey = (String, i128, String, String); +/// The hits VCF's call floor and data lines. +pub type HitRecords = (Floor, Vec>); + +/// The hits VCF, read once: its call floor and its data lines. +#[derive(Debug, Clone)] +pub struct HitsVcf { + /// The path (full) and the path relative to the sample folder. + pub path: String, + pub rel: String, + /// The floor and the records, or the note of a file that cannot be read. + pub read: Result, +} + +impl HitsVcf { + pub fn read(path: String, rel: String) -> HitsVcf { + let read = text::lines(&path) + .map(|lines| { + let mut floor = Floor::default(); + for l in lines.iter().take_while(|l| l.starts_with("##")) { + if l.starts_with("##PGP_CALL_FLOOR=") { + floor_values(l, false, &mut floor); + } + } + let recs = lines + .iter() + .filter(|l| !l.starts_with('#')) + .map(|l| l.split('\t').map(str::to_owned).collect()) + .collect(); + (floor, recs) + }) + .map_err(|e: ReadError| e.unreadable(&path)); + HitsVcf { path, rel, read } + } +} + +/// One hit as `clinvar_row` reads it. +#[derive(Debug, Clone, PartialEq)] +pub struct Hit { + pub stars: u8, + pub chrom: String, + pub pos: i128, + pub reference: String, + pub alt: String, + pub genotype: String, + pub gene: String, + pub significance: String, + pub review_status: String, + pub combined_allele: String, + pub dp: Option, + pub gq: Option, + pub low_confidence: bool, + pub frequency: Option, + pub cdna: Option, +} + +/// gnomAD and rsIDs of one hit from VEP. +#[derive(Debug, Clone, PartialEq)] +pub struct Frequency { + pub exome: Option, + pub genome: Option, + pub rsids: Vec, +} + +impl Hit { + /// `clinvar_row`: an error is the collector's note (a POS that is no + /// integer). + pub fn parse(r: &[String], floor: Floor) -> Result { + let info = if r.len() > 7 { + info_map(&r[7]) + } else { + HashMap::new() + }; + let get = |k: &str| info.get(k).copied().unwrap_or(""); + let rev = get("CLNREVSTAT"); + let genes: Vec<&str> = get("GENEINFO") + .split('|') + .filter(|g| !g.is_empty()) + .map(|g| g.split(':').next().unwrap_or("")) + .collect(); + let gt = gt_of(r); + let fmt = if r.len() > 9 { + format_map(&r[8], &r[9]) + } else { + HashMap::new() + }; + let dp = digits(fmt.get("DP").copied()); + let gq = digits(fmt.get("GQ").copied()); + let pos = text::pos(r)?; + let s = |v: &str| v.replace('_', " "); + let or_dot = |v: String| if v.is_empty() { ".".to_owned() } else { v }; + Ok(Hit { + stars: ReviewStatus::parse(rev.as_bytes()).stars(), + chrom: r[0].clone(), + pos, + reference: r[3].clone(), + alt: r[4].clone(), + genotype: String::from_utf8_lossy(&zygosity(gt.as_bytes())).into_owned(), + gene: or_dot(genes.join(",")), + significance: or_dot(s(get("CLNSIG"))), + review_status: or_dot(s(rev)), + combined_allele: String::from_utf8_lossy(&combined_note( + get("PGP_JOINED_FROM").as_bytes(), + get("PGP_PART_OF").as_bytes(), + get("PGP_ONE_ALLELE").as_bytes(), + gt.as_bytes(), + )) + .into_owned(), + dp, + gq, + low_confidence: gq.is_some_and(|v| v < floor.gq) || dp.is_some_and(|v| v < floor.dp), + frequency: None, + cdna: None, + }) + } + + pub fn key(&self) -> (String, i128, String, String) { + ( + self.chrom.clone(), + self.pos, + self.reference.clone(), + self.alt.clone(), + ) + } + + /// `chrom:pos ref>alt`. + pub fn variant(&self) -> String { + format!( + "{}:{} {}>{}", + self.chrom, self.pos, self.reference, self.alt + ) + } + + fn to_json(&self) -> Value { + let mut m = IndexMap::new(); + let put = |m: &mut IndexMap, k: &str, v: Value| { + m.insert(k.to_owned(), v); + }; + put(&mut m, "stars", Value::Int(self.stars.into())); + put(&mut m, "chrom", Value::Str(self.chrom.clone())); + put(&mut m, "pos", Value::Int(self.pos)); + put(&mut m, "ref", Value::Str(self.reference.clone())); + put(&mut m, "alt", Value::Str(self.alt.clone())); + put(&mut m, "genotype", Value::Str(self.genotype.clone())); + put(&mut m, "gene", Value::Str(self.gene.clone())); + put( + &mut m, + "significance", + Value::Str(self.significance.clone()), + ); + put( + &mut m, + "review_status", + Value::Str(self.review_status.clone()), + ); + put( + &mut m, + "combined_allele", + Value::Str(self.combined_allele.clone()), + ); + put(&mut m, "dp", opt_int(self.dp)); + put(&mut m, "gq", opt_int(self.gq)); + put(&mut m, "low_confidence", Value::Bool(self.low_confidence)); + if let Some(f) = &self.frequency { + let af = py_max(f.exome.into_iter().chain(f.genome)); + put(&mut m, "gnomad_exome_af", opt_f(f.exome)); + put(&mut m, "gnomad_genome_af", opt_f(f.genome)); + put( + &mut m, + "rsids", + Value::Array(f.rsids.iter().cloned().map(Value::Str).collect()), + ); + put(&mut m, "af", opt_f(af)); + put( + &mut m, + "common", + Value::Bool(af.is_some_and(|a| a > COMMON_AF)), + ); + } + if let Some(c) = &self.cdna { + put(&mut m, "cdna", c.clone()); + } + Value::Object(m) + } +} + +/// A hit's genotype as the carrier list words it. +pub fn carrier_genotype(gt: &str, hit: &Hit) -> String { + if half_call(gt.as_bytes()) { + hit.genotype.clone() + } else { + let (copies, ploidy) = allele_copies(gt); + zygosity_word(copies, ploidy).to_owned() + } +} + +/// `(alt copies, ploidy)` of a GT: a missing allele is a copy not called, +/// not a missing copy, so the half of a split call is never hemizygous. +pub fn allele_copies(gt: &str) -> (usize, usize) { + let a: Vec<&str> = gt.split(['/', '|']).filter(|x| !x.is_empty()).collect(); + ( + a.iter().filter(|x| !matches!(**x, "0" | ".")).count(), + a.len(), + ) +} + +pub fn zygosity_word(copies: usize, ploidy: usize) -> &'static str { + if copies == 0 { + "unknown" + } else if ploidy == 1 { + "hemizygous" + } else if copies >= 2 { + "hom" + } else { + "het" + } +} + +pub fn obj<'a>(pairs: impl IntoIterator) -> Value { + Value::Object(pairs.into_iter().map(|(k, v)| (k.to_owned(), v)).collect()) +} + +pub fn opt_int(x: Option) -> Value { + x.map_or(Value::Null, Value::Int) +} + +pub fn opt_f(x: Option) -> Value { + x.map_or(Value::Null, Value::Float) +} + +pub fn opt_s(x: Option<&str>) -> Value { + x.map_or(Value::Null, |s| Value::Str(s.to_owned())) +} + +// --- the cDNA read check, the scan, the deep intronic sites ---------------------- + +/// A row key of a cDNA check table: chrom and the alleles may be missing. +pub type CdnaKey = (Option, i128, Option, Option); + +/// `clinvar_cdna_checks`: the read check of each row, keyed by variant. +pub fn cdna_checks(path: &str) -> Result, String> { + let rows = text::rows(path).map_err(|e| match e { + ReadError::Os(_) => e.unreadable(path), + ReadError::Other(n) => n, + })?; + let mut out = HashMap::new(); + for r in &rows { + let Some(pos) = digits(text::cell(r, "pos")) else { + continue; + }; + let c = |k: &str| text::cell(r, k).map(str::to_owned); + let or = |k: &str, d: &str| { + Value::Str( + text::cell(r, k) + .filter(|v| !v.is_empty()) + .unwrap_or(d) + .to_owned(), + ) + }; + let n = |k: &str| opt_int(digits(text::cell(r, k))); + out.insert( + (c("chrom"), pos, c("ref"), c("alt")), + obj([ + ("check", or("cdna_check", "not_checked")), + ("note", or("cdna_note", "")), + ("alt_reads", n("alt_reads")), + ("alt_clipped", n("alt_clipped")), + ("ref_reads", n("ref_reads")), + ("ref_clipped", n("ref_clipped")), + ( + "construct", + opt_s(text::cell(r, "construct_site").filter(|v| !v.is_empty())), + ), + ]), + ); + } + Ok(out) +} + +/// The check of a variant the table does not hold. +pub fn not_in_table() -> Value { + obj([ + ("check", Value::Str("not_checked".into())), + ("note", Value::Str("not in the read-check table".into())), + ("alt_reads", Value::Null), + ("alt_clipped", Value::Null), + ("ref_reads", Value::Null), + ("ref_clipped", Value::Null), + ]) +} + +pub fn cdna_key(chrom: &str, pos: i128, r: &str, a: &str) -> CdnaKey { + ( + Some(chrom.to_owned()), + pos, + Some(r.to_owned()), + Some(a.to_owned()), + ) +} + +pub fn check_word(c: &Value) -> Option<&str> { + match c { + Value::Object(m) => match m.get("check") { + Some(Value::Str(s)) => Some(s), + _ => None, + }, + _ => None, + } +} + +fn has_construct(c: &Value) -> bool { + matches!(c, Value::Object(m) if m.get("construct").is_some_and(pycompat::value::truthy)) +} + +/// `clinvar_cdna_scan`: the scan's note and loci. +fn cdna_scan(path: &str) -> Result { + let lines = text::lines(path).map_err(|e| e.unreadable(path))?; + let note = lines + .first() + .and_then(|l| l.strip_prefix("# cdna_scan: ")) + .unwrap_or("") + .to_owned(); + let body = lines + .iter() + .filter(|l| !l.starts_with('#')) + .map(|l| Ok::<_, std::io::Error>(l.clone())); + let mut loci = Vec::new(); + for r in pycompat::csv::DictReader::new(body, '\t', pycompat::csv::Quoting::Minimal) { + let r = r.map_err(|e| format!("{}: {e}", e.exception()))?; + let Some(start) = digits(text::cell(&r, "start")) else { + continue; + }; + let int0 = |k: &str| -> Result { + match text::cell(&r, k).filter(|v| !v.is_empty()) { + None => Ok(Value::Int(0)), + Some(v) => crate::int(v) + .map(Value::Int) + .map_err(|e| format!("ValueError: {e}")), + } + }; + loci.push(obj([ + ("chrom", opt_s(text::cell(&r, "chrom"))), + ("start", Value::Int(start)), + ("end", int0("end")?), + ("junctions", int0("junctions")?), + ("reads", int0("reads")?), + ( + "junction_list", + Value::Str(text::cell_or(&r, "junction_list").to_owned()), + ), + ])); + } + Ok(obj([ + ("note", Value::Str(note)), + ("loci", Value::Array(loci)), + ])) +} + +/// `clinvar_deep_intronic`: one row per always-report site. +fn deep_intronic(path: &str) -> Result { + let rows = text::rows(path).map_err(|e| match e { + ReadError::Os(_) => e.unreadable(path), + ReadError::Other(n) => n, + })?; + let mut out = Vec::new(); + for r in &rows { + let Some(pos) = digits(text::cell(r, "pos")) else { + continue; + }; + let or = |k: &str, d: &str| { + Value::Str( + text::cell(r, k) + .filter(|v| !v.is_empty()) + .unwrap_or(d) + .to_owned(), + ) + }; + out.push(obj([ + ("chrom", opt_s(text::cell(r, "chrom"))), + ("pos", Value::Int(pos)), + ("ref", opt_s(text::cell(r, "ref"))), + ("alt", opt_s(text::cell(r, "alt"))), + ("gene", or("gene", ".")), + ("hgvs", or("hgvs", "")), + ("clinvar_id", or("clinvar_id", "")), + ("call", or("call", "no call")), + ("detail", or("detail", "")), + ])); + } + Ok(Value::Array(out)) +} + +// --- gnomAD AF and rsIDs from step 13's VEP output ----------------------------------- + +/// Where the frequencies came from, or why there are none. +#[derive(Debug, Clone, PartialEq)] +pub struct FrequencySource { + pub source: Option, + pub note: Option, +} + +/// The VEP output's frequencies of every hit, in one pass per contig over +/// the hits sorted by position: one indexed query per hit position, served +/// by the reader's block cache when hits share a block. +pub fn vep_frequencies(vep: &str, keys: &[VarKey]) -> Result, String> { + let mut fmt: Option> = None; + { + let mut lines = og_io::Lines::new( + og_io::open(vep).map_err(|e| crate::os_error(&e, vep))?, + og_io::Utf8::Replace, + ); + let mut line = String::new(); + while lines.read_line(&mut line).map_err(|e| e.to_string())? { + if !line.starts_with('#') { + break; + } + if line.starts_with("##INFO= = keys.iter().collect(); + sorted.sort(); + sorted.dedup(); + let mut out = HashMap::new(); + if sorted.is_empty() { + return Ok(out); + } + let mut reader = + og_io::tabix::Reader::with_index(vep, format!("{vep}.tbi")).map_err(|e| e.to_string())?; + for key in sorted { + let (chrom, pos, reference, alt) = key; + let Ok(p) = u64::try_from(*pos) else { continue }; + if p == 0 { + continue; + } + let lines: Vec> = reader + .query(chrom, p - 1, p) + .collect::>() + .map_err(|e| e.to_string())?; + for raw in lines { + let line = String::from_utf8_lossy(&raw); + let r: Vec<&str> = line.split('\t').collect(); + if r.len() < 3 || r[1].parse::().ok() != Some(*pos) { + continue; + } + let alts: Vec<&str> = if r.len() > 7 { + r[4].split(',').collect() + } else { + Vec::new() + }; + if r.get(3) != Some(&reference.as_str()) || !alts.contains(&alt.as_str()) { + continue; + } + // VEP's Allele drops the base an indel's alleles share. + let first = |s: &str| s.chars().next(); + let snv = reference.chars().count() == 1 && alts.iter().all(|a| a.chars().count() == 1); + let allele = if snv || alts.iter().any(|a| first(a) != first(reference)) { + alt.clone() + } else { + let rest: String = alt.chars().skip(1).collect(); + if rest.is_empty() { + "-".to_owned() + } else { + rest + } + }; + let mut got = [None, None]; + let mut ids: Vec = Vec::new(); + for tr in info_map(r[7]).get("CSQ").copied().unwrap_or("").split(',') { + let v: Vec<&str> = tr.split('|').collect(); + if v.len() != fmt.len() || (alts.len() > 1 && v[allele_ix] != allele) { + continue; + } + for (i, c) in cols.iter().enumerate() { + if let Some(c) = c + && got[i].is_none() + { + got[i] = py_max( + v[*c] + .split('&') + .filter_map(|x| pycompat::num::parse_float(x).ok()), + ); + } + } + if let Some(e) = ev { + for x in v[e].split('&') { + if is_rsid(x) { + let x = x.to_lowercase(); + if !ids.contains(&x) { + ids.push(x); + } + } + } + } + } + ids.sort_by_key(|x| x[2..].parse::().unwrap_or(u128::MAX)); + out.insert( + key.clone(), + Frequency { + exome: got[0], + genome: got[1], + rsids: ids, + }, + ); + } + } + Ok(out) +} + +/// `(?i)rs\d+`, the whole field. +fn is_rsid(x: &str) -> bool { + let b = x.as_bytes(); + b.len() > 2 && b[..2].eq_ignore_ascii_case(b"rs") && b[2..].iter().all(u8::is_ascii_digit) +} + +// --- the section -------------------------------------------------------------------- + +/// The data of the ClinVar section. +#[derive(Debug, Clone, PartialEq)] +pub struct ClinVar { + pub hits: Vec, + pub floor: Floor, + pub frequency: FrequencySource, + pub cdna: bool, + pub cdna_scan: Option, + pub deep_intronic: Option, +} + +impl ClinVar { + /// `sec_clinvar` over the hits read once. `side(rel)` is the file of + /// another of step 06's or 13's outputs, found in the folder. + pub fn read( + hits: &HitsVcf, + dir: &str, + side: &dyn Fn(&str) -> Option<(String, String)>, + s: &str, + ) -> Result { + let (floor, recs) = hits.read.clone()?; + let mut rows = Vec::new(); + for r in recs.iter().filter(|r| r.len() >= 8) { + rows.push(Hit::parse(r, floor)?); + } + rows.sort_by(|a, b| { + b.stars + .cmp(&a.stars) + .then_with(|| a.chrom.cmp(&b.chrom)) + .then_with(|| a.pos.cmp(&b.pos)) + }); + let frequency = frequencies(dir, side, s, &mut rows); + let mut cdna = false; + if let Some((p, _)) = side(&format!("clinvar/{s}_clinvar_cdna_check.tsv")) { + let table = cdna_checks(&p)?; + for x in &mut rows { + x.cdna = Some( + table + .get(&cdna_key(&x.chrom, x.pos, &x.reference, &x.alt)) + .cloned() + .unwrap_or_else(not_in_table), + ); + } + cdna = true; + } + let cdna_scan = match side(&format!("clinvar/{s}_clinvar_cdna_scan.tsv")) { + Some((p, _)) => Some(cdna_scan(&p)?), + None => None, + }; + let deep_intronic = match side(&format!("clinvar/{s}_clinvar_deep_intronic.tsv")) { + Some((p, _)) => Some(deep_intronic(&p)?), + None => None, + }; + Ok(ClinVar { + hits: rows, + floor, + frequency, + cdna, + cdna_scan, + deep_intronic, + }) + } + + pub fn to_json(&self) -> Value { + let sure: Vec<&Hit> = self.hits.iter().filter(|x| !x.low_confidence).collect(); + let by = (0..=4u8) + .rev() + .map(|k| { + ( + k.to_string(), + Value::Int(sure.iter().filter(|x| x.stars == k).count() as i128), + ) + }) + .collect(); + let mut m: IndexMap = IndexMap::new(); + m.insert("count".into(), Value::Int(sure.len() as i128)); + m.insert("by_stars".into(), Value::Object(by)); + m.insert( + "hits".into(), + Value::Array(self.hits.iter().map(Hit::to_json).collect()), + ); + m.insert( + "low_confidence".into(), + Value::Int((self.hits.len() - sure.len()) as i128), + ); + m.insert("call_floor".into(), self.floor.to_json()); + m.insert( + "frequency".into(), + obj([ + ("source", opt_s(self.frequency.source.as_deref())), + ("note", opt_s(self.frequency.note.as_deref())), + ("common_af", Value::Float(COMMON_AF)), + ]), + ); + if self.cdna { + let checks: Vec<&Value> = self.hits.iter().filter_map(|x| x.cdna.as_ref()).collect(); + m.insert( + "cdna_flagged".into(), + Value::Int( + checks + .iter() + .filter(|c| check_word(c) == Some("flagged")) + .count() as i128, + ), + ); + m.insert( + "construct_sites".into(), + Value::Int(checks.iter().filter(|c| has_construct(c)).count() as i128), + ); + } + if let Some(v) = &self.cdna_scan { + m.insert("cdna_scan".into(), v.clone()); + } + if let Some(v) = &self.deep_intronic { + m.insert("deep_intronic".into(), v.clone()); + } + Value::Object(m) + } +} + +/// `clinvar_frequencies`: puts the frequencies on each hit and says where +/// they came from. +fn frequencies( + dir: &str, + side: &dyn Fn(&str) -> Option<(String, String)>, + s: &str, + rows: &mut [Hit], +) -> FrequencySource { + let vep = side(&format!("vep/{s}_vep.vcf.gz")); + let _ = dir; + let mut note = None; + let mut found = HashMap::new(); + match &vep { + None => note = Some("no VEP output (step 13) in this folder".to_owned()), + Some((p, rel)) if !std::path::Path::new(&format!("{p}.tbi")).exists() => { + note = Some(format!("{rel} has no .tbi index")); + } + Some((p, rel)) => { + let keys: Vec<_> = rows.iter().map(Hit::key).collect(); + match vep_frequencies(p, &keys) { + Ok(f) => found = f, + Err(e) => note = Some(format!("{rel} could not be read: {e}")), + } + } + } + for x in rows.iter_mut() { + x.frequency = Some(found.get(&x.key()).cloned().unwrap_or(Frequency { + exome: None, + genome: None, + rsids: Vec::new(), + })); + } + FrequencySource { + source: if note.is_none() { + vep.map(|(_, rel)| rel) + } else { + None + }, + note, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn floor_takes_the_last_value_of_each_key() { + let mut f = Floor::default(); + floor_values("##PGP_CALL_FLOOR=GQ=20,DP=8,GQ=25", false, &mut f); + assert_eq!(f, Floor { gq: 25, dp: 8 }); + let mut f = Floor::default(); + floor_values("##PGP_CALL_FLOOR=XGQ=5,DP=7", true, &mut f); + assert_eq!(f, Floor { gq: 30, dp: 7 }); + } + + #[test] + fn copies_and_words() { + assert_eq!(allele_copies("0/1"), (1, 2)); + assert_eq!(allele_copies("1|1"), (2, 2)); + assert_eq!(allele_copies("1"), (1, 1)); + assert_eq!(allele_copies("1/."), (1, 2)); + assert_eq!(allele_copies(""), (0, 0)); + assert_eq!(zygosity_word(1, 1), "hemizygous"); + assert_eq!(zygosity_word(0, 2), "unknown"); + } + + #[test] + fn rsid_fields() { + assert!(is_rsid("rs123")); + assert!(is_rsid("RS9")); + assert!(!is_rsid("rs")); + assert!(!is_rsid("COSV1")); + } +} diff --git a/rust/crates/og-summary/src/collect/cpsr.rs b/rust/crates/og-summary/src/collect/cpsr.rs new file mode 100644 index 000000000..ffd178d9c --- /dev/null +++ b/rust/crates/og-summary/src/collect/cpsr.rs @@ -0,0 +1,264 @@ +//! Cancer predisposition (step 17, CPSR): the class counts, and CPSR's own +//! class beside the final class and ClinVar's, so a final P/LP class that +//! ClinVar set over CPSR's own benign call is not hidden. + +use super::clinvar::{CdnaKey, Floor, cdna_checks, check_word, not_in_table, opt_int, opt_s}; +use super::text::{self, cell, digits, stripped}; +use indexmap::IndexMap; +use og_bam::cdna::{cpsr_variant, norm_class}; +use pycompat::csv::DictRow; +use pycompat::json::Value; +use std::collections::HashMap; + +const PLP: [&str; 2] = ["pathogenic", "likely pathogenic"]; +const CLASS_COLUMNS: [&str; 3] = [ + "CLASSIFICATION", + "FINAL_CLASSIFICATION", + "CPSR_CLASSIFICATION", +]; + +#[derive(Debug, Clone, PartialEq)] +pub struct Cpsr { + pub html_report: bool, + /// The class column and the counts, when the table was read. + pub table: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Table { + /// `None`: the table has no class column. + pub column: Option, + pub classification: IndexMap, + pub compare: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Compare { + pub plp: Vec, + pub disagreements: Vec, + pub own_benign: usize, + pub own: Option, + pub floor_dp: i128, + pub cdna: bool, +} + +/// Where the inputs beside the CPSR table are. +pub struct Inputs { + pub html: Option, + /// The classification table (full path). + pub tsv: Option, + /// Step 06's floor, the default without a hits VCF; an error is the + /// note of a hits VCF that cannot be read. + pub floor: Result, + /// Step 17's cDNA read check (full path). + pub cdna: Option, +} + +impl Cpsr { + pub fn read(i: &Inputs) -> Result { + let html_report = i.html.is_some(); + let Some(tsv) = &i.tsv else { + return Ok(Cpsr { + html_report, + table: None, + }); + }; + let rows = text::rows(tsv).map_err(|e| match e { + text::ReadError::Os(_) => e.unreadable(tsv), + text::ReadError::Other(n) => n, + })?; + // `rows[0].keys()`: the header, and the None key of a long first row. + let cols: Vec<&str> = rows + .first() + .map(|r| r.iter().map(|(k, _)| k).collect()) + .unwrap_or_default(); + let col = CLASS_COLUMNS.iter().find(|c| cols.contains(c)).copied(); + let Some(col) = col else { + return Ok(Cpsr { + html_report, + table: Some(Table { + column: None, + classification: IndexMap::new(), + compare: None, + }), + }); + }; + let mut counts: IndexMap = IndexMap::new(); + for r in &rows { + let k = cell(r, col).filter(|v| !v.is_empty()).unwrap_or("."); + *counts.entry(k.to_owned()).or_default() += 1; + } + counts.sort_by(|_, a, _, b| b.cmp(a)); + let compare = compare(i, &rows, &cols, col)?; + Ok(Cpsr { + html_report, + table: Some(Table { + column: Some(col.to_owned()), + classification: counts, + compare: Some(compare), + }), + }) + } + + pub fn to_json(&self) -> Value { + let mut m: IndexMap = IndexMap::new(); + m.insert("html_report".into(), Value::Bool(self.html_report)); + let Some(t) = &self.table else { + return Value::Object(m); + }; + m.insert("classification_column".into(), opt_s(t.column.as_deref())); + if t.column.is_some() { + m.insert( + "classification".into(), + Value::Object( + t.classification + .iter() + .map(|(k, v)| (k.clone(), Value::Int(*v as i128))) + .collect(), + ), + ); + } + if let Some(c) = &t.compare { + let field = |x: &Value, k: &str| match x { + Value::Object(m) => m.get(k).cloned(), + _ => None, + }; + let low = c + .plp + .iter() + .filter(|x| field(x, "low_confidence") == Some(Value::Bool(true))) + .count(); + m.insert("plp".into(), Value::Array(c.plp.clone())); + m.insert( + "disagreements".into(), + Value::Array(c.disagreements.clone()), + ); + m.insert( + "final_plp_own_benign".into(), + Value::Int(c.own_benign as i128), + ); + m.insert("own_class_column".into(), opt_s(c.own.as_deref())); + m.insert("low_confidence".into(), Value::Int(low as i128)); + m.insert("call_floor_dp".into(), Value::Int(c.floor_dp)); + if c.cdna { + let flagged = c + .plp + .iter() + .filter(|x| field(x, "cdna").as_ref().and_then(check_word) == Some("flagged")) + .count(); + m.insert("cdna_flagged".into(), Value::Int(flagged as i128)); + } + } + Value::Object(m) + } +} + +/// `cpsr_compare`. +fn compare(i: &Inputs, rows: &[DictRow], cols: &[&str], col: &str) -> Result { + let own = (cols.contains(&"CPSR_CLASSIFICATION") && col != "CPSR_CLASSIFICATION") + .then_some("CPSR_CLASSIFICATION"); + let floor = i.floor.clone()?; + let cdna: Option> = match &i.cdna { + Some(p) => Some(cdna_checks(p)?), + None => None, + }; + let has_clinvar_col = cols.contains(&"CLINVAR_CLASSIFICATION"); + let (mut plp, mut disagree, mut own_benign) = (Vec::new(), Vec::new(), 0); + for r in rows { + let final_ = norm_class(cell(r, col)); + let mine = own.map(|o| norm_class(cell(r, o))).unwrap_or_default(); + let authority = stripped(r, "ASSERTION_AUTHORITY"); + let clinvar_raw = if has_clinvar_col { + stripped(r, "CLINVAR_CLASSIFICATION") + } else if authority.to_lowercase() == "clinvar" { + stripped(r, col) + } else { + "" + }; + let clinvar = norm_class(Some(clinvar_raw)); + let is = |c: &str| PLP.contains(&c); + if is(&final_) && matches!(mine.as_str(), "benign" | "likely benign") { + own_benign += 1; + } + let is_plp = is(&final_) || is(&mine); + let differ = + own.is_some() && !clinvar.is_empty() && !mine.is_empty() && is(&clinvar) != is(&mine); + if !(is_plp || differ) { + continue; + } + let v = cpsr_variant(cell(r, "VAR_ID"), cell(r, "GENOMIC_CHANGE")) + .map_err(|e| format!("ValueError: {e}"))?; + let dp = digits(Some(stripped(r, "DP_CONTROL"))); + let or_dot = + |x: Option<&str>| Value::Str(x.filter(|v| !v.is_empty()).unwrap_or(".").to_owned()); + let mut x: IndexMap = IndexMap::new(); + x.insert( + "variant".into(), + match &v { + Some(h) => Value::Str(format!("{}:{} {}>{}", h.chrom, h.pos, h.reference, h.alt)), + None => or_dot(cell(r, "GENOMIC_CHANGE")), + }, + ); + x.insert("gene".into(), or_dot(cell(r, "SYMBOL"))); + x.insert("final".into(), or_dot(cell(r, col))); + x.insert( + "cpsr".into(), + own.map_or(Value::Null, |o| or_dot(cell(r, o))), + ); + x.insert( + "clinvar".into(), + if clinvar.is_empty() { + Value::Null + } else { + Value::Str(clinvar_raw.to_owned()) + }, + ); + x.insert( + "authority".into(), + opt_s(Some(authority).filter(|a| !a.is_empty())), + ); + x.insert( + "acmg_codes".into(), + opt_s(Some(stripped(r, "ACMG_CODE")).filter(|a| !a.is_empty())), + ); + x.insert("dp".into(), opt_int(dp)); + x.insert( + "low_confidence".into(), + Value::Bool(dp.is_some_and(|d| d < floor.dp)), + ); + if let Some(t) = &cdna + && is_plp + { + let found = v.as_ref().and_then(|h| { + t.get(&( + Some(h.chrom.clone()), + i128::from(h.pos), + Some(h.reference.clone()), + Some(h.alt.clone()), + )) + }); + x.insert( + "cdna".into(), + found + .filter(|c| pycompat::value::truthy(c)) + .cloned() + .unwrap_or_else(not_in_table), + ); + } + let x = Value::Object(x); + if differ { + disagree.push(x.clone()); + } + if is_plp { + plp.push(x); + } + } + Ok(Compare { + plp, + disagreements: disagree, + own_benign, + own: own.map(str::to_owned), + floor_dp: floor.dp, + cdna: cdna.is_some(), + }) +} diff --git a/rust/crates/og-summary/src/collect/group_b.rs b/rust/crates/og-summary/src/collect/group_b.rs new file mode 100644 index 000000000..cb0ba8a1e --- /dev/null +++ b/rust/crates/og-summary/src/collect/group_b.rs @@ -0,0 +1,284 @@ +//! Group b of `og summary collect`: the clinical sections. ClinVar, the +//! carrier list, PharmCAT, CPIC, pypgx, Cyrius, CPSR, the clinical filter +//! with its ACMG secondary-findings tier, slivar, and the `cyp2d6` block. +//! +//! Each input is read once: the ClinVar hits VCF feeds the ClinVar +//! section, the ACMG tier, the carrier list and CPSR's call floor; the +//! clinical VCF feeds the record count and the ACMG tier; AnnotSV's table +//! and step 35's copy-number tables feed both two-hit readings. The +//! ClinVar section (with its VEP lookups) and the clinical section (with +//! the clinical VCF) are read on their own threads. + +use super::acmg::{self, ClinicalVcf}; +use super::carrier::Carrier; +use super::clinical::{Clinical, Slivar}; +use super::clinvar::{ClinVar, Floor, HitsVcf}; +use super::cpsr::{self, Cpsr}; +use super::model::{Manifest, RunStatus, Section, Spec, Summary}; +use super::pgx::{Cpic, Cyp2d6, Cyrius, PharmCat, Pypgx}; +use super::scan::FolderIndex; +use super::sections::{Data, SampleQc}; +use super::two_hits::{self, DeletionInputs}; +use super::{Input, Read, read_at, read_optional, with_mtime}; +use std::path::Path; + +/// The sections `og summary collect --only b` reads, in summary.json's order. +pub const GROUP_B: [Spec; 9] = [ + Spec { + key: "clinvar", + title: "ClinVar screen", + step: "06", + }, + Spec { + key: "carrier", + title: "Carrier list", + step: "06", + }, + Spec { + key: "pharmcat", + title: "PharmCAT", + step: "07", + }, + Spec { + key: "cpic", + title: "CPIC drug-gene recommendations", + step: "27", + }, + Spec { + key: "pypgx", + title: "pypgx", + step: "32", + }, + Spec { + key: "cyrius", + title: "Cyrius (CYP2D6)", + step: "21", + }, + Spec { + key: "cpsr", + title: "Cancer predisposition (CPSR)", + step: "17", + }, + Spec { + key: "clinical", + title: "Clinical variant filter", + step: "23", + }, + Spec { + key: "slivar", + title: "Variant prioritization (slivar)", + step: "31", + }, +]; + +/// Reads the folder into the summary of group b. An error is a run status +/// or a manifest that is there and cannot be read. +pub fn collect(input: &Input) -> Result { + let dir = Path::new(&input.sample_dir); + let d = input.sample_dir.as_str(); + let index = FolderIndex::walk(dir); + let run = + read_optional(&dir.join("logs").join("run_status.tsv"))?.map(|t| RunStatus::parse(&t)); + let manifest = read_optional(&dir.join("run_manifest.tsv"))?.map(|t| Manifest::parse(&t)); + let s = pycompat::mtime::escape(&input.sample); + let raw = input.sample.as_str(); + let file = |rel: &str| -> (String, String) { (crate::join(d, rel), rel.to_owned()) }; + let find = |pats: &[String]| index.first(pats).map(file); + let one = |p: String| find(&[p]); + + // The shared inputs, each read once. + let hits = one(format!("clinvar/{s}_clinvar_hits.vcf")).map(|(p, rel)| HitsVcf::read(p, rel)); + let clinical_table = one(format!("clinical/{s}_clinical_summary.tsv")); + let clinical_vcf_at = one(format!("clinical/{s}_clinical.vcf.gz")); + let deletions = DeletionInputs { + annotsv: two_hits::read_annotsv( + one(format!("annotsv/{s}_sv_class4_5_checked.tsv")), + one(format!("sv_merged/{s}_sv_consensus.vcf.gz")).map(|(p, _)| p), + ), + step35: two_hits::read_step35(d, raw, &input.assets), + }; + let side = |rel: &str| one(rel.to_owned()); + + let (clinvar, clinical, rest) = std::thread::scope(|scope| { + let clinvar = scope.spawn(|| -> Read { + let Some(h) = &hits else { return Ok(None) }; + let data = ClinVar::read(h, d, &side, &s)?; + with_mtime(&h.path, h.rel.clone(), Data::ClinVar(data)) + }); + let clinical = scope.spawn(|| -> Read { + let at = clinical_table.clone().or_else(|| clinical_vcf_at.clone()); + let Some((p, rel)) = at else { return Ok(None) }; + let vcf = clinical_vcf_at.as_ref().map(|(v, _)| ClinicalVcf::read(v)); + let tier = acmg::Inputs { + assets: &input.assets, + hits: hits.as_ref().map(|h| &h.read), + clinical: vcf.as_ref(), + deletions: &deletions, + }; + let data = Clinical::read( + clinical_table.as_ref().map(|(p, _)| p.as_str()), + vcf.as_ref(), + &tier, + )?; + with_mtime(&p, rel, Data::Clinical(data)) + }); + let rest = read_rest(input, &s, &index, &find, &hits, &deletions); + let join = |h: std::thread::ScopedJoinHandle<'_, Read>| { + h.join() + .unwrap_or_else(|_| Err("internal error: the section reader stopped".into())) + }; + (join(clinvar), join(clinical), rest) + }); + let [carrier, pharmcat, cpic, pypgx, cyrius, cpsr, slivar] = rest; + let reads = [ + clinvar, carrier, pharmcat, cpic, pypgx, cyrius, cpsr, clinical, slivar, + ]; + let sections: Vec
= GROUP_B + .iter() + .zip(reads) + .map(|(spec, r)| Section::new(*spec, r, run.as_ref())) + .collect(); + let sec = |k: &str| sections.iter().find(|x| x.spec.key == k); + let cyp2d6 = match (sec("cpic"), sec("pypgx"), sec("cyrius")) { + (Some(c), Some(p), Some(y)) => Some(Cyp2d6::new( + c, + p, + y, + one(format!("pgx_consensus/{s}_pgx_consensus.tsv")), + d, + )), + _ => None, + }; + + // The run block is group a's: the declared sex from the command line, + // the run status, the manifest or step 33's table, the first that has one. + let qc_declared = one(format!("qc/{s}_sample_qc.tsv")).and_then(|(p, _)| { + let rows = super::sections::read_tsv(&p).ok()?; + SampleQc::read(&p, &rows) + .ok()? + .declared_sex() + .map(str::to_owned) + }); + let declared_sex = [ + input.declared_sex.clone(), + run.as_ref().and_then(|r| r.declared_sex.clone()), + manifest + .as_ref() + .and_then(|m| m.value("run", "declared_sex").map(str::to_owned)), + qc_declared, + ] + .into_iter() + .flatten() + .find(|v| !v.is_empty()); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0.0, |d| d.as_secs_f64()); + Ok(Summary { + sample: input.sample.clone(), + generated_utc: super::time::utc_minute(now).unwrap_or_default(), + run, + manifest, + declared_sex, + sections, + cyp2d6, + }) +} + +/// Finds a section's file: the full path and the path in the folder. +type Find<'a> = dyn Fn(&[String]) -> Option<(String, String)> + 'a; + +/// The seven sections read on the calling thread, in order. +fn read_rest( + input: &Input, + s: &str, + index: &FolderIndex, + find: &Find, + hits: &Option, + deletions: &DeletionInputs, +) -> [Read; 7] { + let one = |p: String| find(&[p]); + let carrier = match hits { + None => Ok(None), + Some(h) => Carrier::read(&input.assets, &h.read, deletions) + .and_then(|c| with_mtime(&h.path, h.rel.clone(), Data::Carrier(c))), + }; + let pharmcat = pharmcat_at(index, input, s) + .and_then(|at| read_at(at, |p| PharmCat::read(p).map(Data::PharmCat))); + let cpic = read_at(one(format!("cpic/{s}_phenotypes.tsv")), |p| { + Cpic::read(p, &input.sample).map(Data::Cpic) + }); + let pypgx = read_at(one(format!("pypgx/{s}_pypgx_summary.tsv")), |p| { + Pypgx::read(p, &input.sample).map(Data::Pypgx) + }); + let cyrius = read_at( + find(&[ + format!("cyrius/{s}_cyp2d6.tsv"), + "cyrius/*_cyp2d6.tsv".to_owned(), + ]), + |p| Cyrius::read(p).map(Data::Cyrius), + ); + let html = find(&[ + format!("cpsr/{s}.cpsr.grch38.html"), + "cpsr/*.cpsr.*.html".to_owned(), + ]); + let tsv = find(&[ + format!("cpsr/{s}.cpsr.grch38.classification.tsv.gz"), + "cpsr/*.cpsr.*.classification.tsv.gz".to_owned(), + ]); + let cpsr = read_at(tsv.clone().or_else(|| html.clone()), |_| { + let floor = match hits { + None => Ok(Floor::default()), + Some(h) => h.read.as_ref().map(|(f, _)| *f).map_err(Clone::clone), + }; + Cpsr::read(&cpsr::Inputs { + html: html.as_ref().map(|(p, _)| p.clone()), + tsv: tsv.as_ref().map(|(p, _)| p.clone()), + floor, + cdna: one(format!("cpsr/{s}_cpsr_cdna_check.tsv")).map(|(p, _)| p), + }) + .map(Data::Cpsr) + }); + let slivar_table = one(format!("slivar/{s}_slivar_summary.tsv")); + let slivar_vcf = if slivar_table.is_none() { + one(format!("slivar/{s}_prioritized.vcf.gz")) + } else { + None + }; + let slivar = read_at(slivar_table.clone().or(slivar_vcf.clone()), |_| { + Slivar::read( + slivar_table.as_ref().map(|(p, _)| p.as_str()), + slivar_vcf.as_ref().map(|(p, _)| p.as_str()), + &input.sample, + ) + .map(Data::Slivar) + }); + [carrier, pharmcat, cpic, pypgx, cyrius, cpsr, slivar] +} + +/// PharmCAT's report: the newest of the files its four names match. +fn pharmcat_at( + index: &FolderIndex, + input: &Input, + s: &str, +) -> Result, String> { + let d = input.sample_dir.as_str(); + let found: Vec<&str> = [ + format!("pharmcat/{s}.report.json"), + "pharmcat/*.report.json".to_owned(), + format!("vcf/{s}.report.json"), + "vcf/*.report.json".to_owned(), + ] + .iter() + .flat_map(|p| index.all(p)) + .collect(); + let paths: Vec<(String, &str)> = found + .iter() + .map(|rel| (crate::join(d, rel), *rel)) + .collect(); + let newest = pycompat::mtime::newest(paths.iter().map(|(p, _)| p.as_str())) + .map_err(|e| e.to_string())?; + Ok(newest.map(|p| { + let rel = paths.iter().find(|(q, _)| q == p).map_or("", |(_, r)| r); + (p.to_owned(), rel.to_owned()) + })) +} diff --git a/rust/crates/og-summary/src/collect/model.rs b/rust/crates/og-summary/src/collect/model.rs index 689d8c087..c6d6e22a3 100644 --- a/rust/crates/og-summary/src/collect/model.rs +++ b/rust/crates/og-summary/src/collect/model.rs @@ -295,6 +295,8 @@ pub struct Summary { pub manifest: Option, pub declared_sex: Option, pub sections: Vec
, + /// The CYP2D6 callers side by side (group b). + pub cyp2d6: Option, } impl Summary { @@ -334,6 +336,9 @@ impl Summary { .collect(), ), ); + if let Some(c) = &self.cyp2d6 { + m.insert("cyp2d6".into(), c.to_json()); + } Value::Object(m) } } diff --git a/rust/crates/og-summary/src/collect/pgx.rs b/rust/crates/og-summary/src/collect/pgx.rs new file mode 100644 index 000000000..458660493 --- /dev/null +++ b/rust/crates/og-summary/src/collect/pgx.rs @@ -0,0 +1,618 @@ +//! The pharmacogenomics sections: PharmCAT (step 07), the CPIC table (step +//! 27), pypgx (step 32) and Cyrius (step 21), and the summary's `cyp2d6` +//! block, which sets the three CYP2D6 callers side by side by step 36's +//! rule (`og_steps::pgx_rules`). + +use super::clinvar::{obj, opt_int, opt_s}; +use super::model::{Section, State}; +use super::sections::Data; +use super::text::{self, cell, stripped}; +use indexmap::IndexMap; +use og_steps::pgx_rules::{NO_CALL, is_call, same_diplotype}; +use og_steps::pypgx_summary::{LOW_GQ, NOT_CALLABLE}; +use pycompat::csv::DictRow; +use pycompat::json::Value; +use pycompat::text::{lower, strip, upper}; + +/// The label of pypgx's CYP2D6 call when it is the only usable one. +pub const PROBABLE: &str = "probable (one caller: pypgx)"; +/// The rows of the CYP2D6 depth check the reports show. +pub const DEPTH_CHECK_KEYS: [&str; 5] = [ + "status", + "gene_mapq1_share", + "gene_mapq20_share", + "body_mapq1_share", + "body_mapq20_share", +]; + +fn rows_or_note(path: &str) -> Result, String> { + text::rows(path).map_err(|e| e.note()) +} + +/// `row.get(key, "")`: `""` without the column, `None` for a short row. +fn get_default(r: &DictRow, key: &str) -> Value { + match r.get(key) { + None => Value::Str(String::new()), + Some(v) => opt_s(v), + } +} + +/// `row[key]` as JSON: a `KeyError` note without the column. +fn need(r: &DictRow, key: &str) -> Result { + text::need(r, key).map(opt_s) +} + +// --- PharmCAT ------------------------------------------------------------------------ + +#[derive(Debug, Clone, PartialEq)] +pub struct PharmCat { + pub version: String, + pub html_report: bool, +} + +impl PharmCat { + pub fn read(path: &str) -> Result { + let raw = std::fs::read(path) + .map_err(|e| format!("{}: {}", crate::basename(path), crate::os_error(&e, path)))?; + let body = String::from_utf8(raw) + .map_err(|e| format!("{}: 'utf-8' codec can't decode: {e}", crate::basename(path)))?; + let data = + pycompat::json::loads(&body).map_err(|e| format!("{}: {e}", crate::basename(path)))?; + let version = match &data { + Value::Object(m) => match m.get("pharmcatVersion") { + None => String::new(), + Some(v) => pycompat::value::py_str(v).map_err(|e| e.to_string())?, + }, + _ => String::new(), + }; + let html = path.strip_suffix(".json").map(|p| format!("{p}.html")); + Ok(PharmCat { + version, + html_report: html.is_some_and(crate::is_file), + }) + } + + pub fn to_json(&self) -> Value { + obj([ + ("version", Value::Str(self.version.clone())), + ("html_report", Value::Bool(self.html_report)), + ]) + } +} + +// --- CPIC ---------------------------------------------------------------------------- + +#[derive(Debug, Clone, PartialEq)] +pub struct CpicGene { + pub gene: Value, + pub diplotype: Value, + pub phenotype: Value, + pub status: Value, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Cpic { + pub genes: Vec, + pub parse_failed: bool, + pub warnings: Vec, +} + +impl Cpic { + /// `sec_cpic`: the phenotype table and, beside it, the recommendation + /// text's parse failure and warnings. + pub fn read(path: &str, s: &str) -> Result { + let genes: Vec = rows_or_note(path)? + .iter() + .map(|r| CpicGene { + gene: get_default(r, "Gene"), + diplotype: get_default(r, "Diplotype"), + phenotype: get_default(r, "Phenotype"), + status: get_default(r, "Status"), + }) + .collect(); + let rec = crate::join( + text::dirname(path), + &format!("{s}_cpic_recommendations.txt"), + ); + let (mut warnings, mut failed) = (Vec::new(), false); + if crate::is_file(&rec) { + for line in text::lines(&rec).map_err(|e| e.note())? { + if line.contains("PARSING FAILED") { + failed = true; + } + let t = strip(&line, None); + if let Some(w) = t.strip_prefix("WARNING:") { + warnings.push(strip(w, None).to_owned()); + } + } + } + let parse_failed = failed || genes.is_empty(); + Ok(Cpic { + genes, + parse_failed, + warnings, + }) + } + + fn count(&self, status: &str) -> Value { + Value::Int( + self.genes + .iter() + .filter(|g| matches!(&g.status, Value::Str(s) if s == status)) + .count() as i128, + ) + } + + /// The CYP2D6 row, if the table has one. + pub fn cyp2d6(&self) -> Option<&CpicGene> { + self.genes + .iter() + .find(|g| matches!(&g.gene, Value::Str(s) if s == "CYP2D6")) + } + + pub fn to_json(&self) -> Value { + obj([ + ( + "genes", + Value::Array( + self.genes + .iter() + .map(|g| { + obj([ + ("gene", g.gene.clone()), + ("diplotype", g.diplotype.clone()), + ("phenotype", g.phenotype.clone()), + ("status", g.status.clone()), + ]) + }) + .collect(), + ), + ), + ("non_normal", self.count("non-normal")), + ("ambiguous", self.count("ambiguous")), + ("not_called", self.count("not called")), + ("unclassified", self.count("unclassified")), + ("parse_failed", Value::Bool(self.parse_failed)), + ( + "warnings", + Value::Array(self.warnings.iter().cloned().map(Value::Str).collect()), + ), + ]) + } +} + +// --- pypgx --------------------------------------------------------------------------- + +#[derive(Debug, Clone, PartialEq)] +pub struct Pypgx { + pub genes_total: usize, + pub genes_called: usize, + pub cyp2d6: String, + /// Gene (as JSON's key text) to the copy-number call or why none. + pub cnv_call: IndexMap, + pub not_callable: Vec, + pub low_gq: Vec, + /// phenotype, defining_filter, quality of the CYP2D6 row. + pub cyp2d6_evidence: Option<[String; 3]>, + pub depth_check: Option>, + /// conflicts, one_tool_only, concordant. + pub comparison: Option<[usize; 3]>, +} + +/// A JSON object key from a cell that may be `None` (`json.dumps` writes `null`). +fn key_text(v: Option<&str>) -> String { + v.unwrap_or("null").to_owned() +} + +impl Pypgx { + pub fn read(path: &str, s: &str) -> Result { + let rows = rows_or_note(path)?; + let called = rows + .iter() + .filter(|r| !NO_CALL.contains(&lower(stripped(r, "Diplotype")).as_str())) + .count(); + let cyp_row = rows.iter().find(|r| cell(r, "Gene") == Some("CYP2D6")); + let cyp = cyp_row.and_then(|r| cell(r, "Diplotype")).unwrap_or(""); + let mut cnv_call = IndexMap::new(); + for r in &rows { + let status = stripped(r, "Status"); + if upper(stripped(r, "Source")) != "BAM" || status == NOT_CALLABLE { + continue; + } + let v = stripped(r, "CNV_call"); + let gene = key_text(text::need(r, "Gene")?); + let call = if !matches!(v, "" | ".") && !v.starts_with("N/A") { + v.to_owned() + } else if lower(status) == "failed" || upper(stripped(r, "Diplotype")) == "FAILED" { + "not called: pypgx failed".to_owned() + } else if status.contains("depth check") || v.contains("depth check") { + "not called: the CYP2D6 depth check failed".to_owned() + } else { + "not called: pypgx gave no copy-number call".to_owned() + }; + cnv_call.insert(gene, call); + } + let mut not_callable = Vec::new(); + for r in rows + .iter() + .filter(|r| stripped(r, "Status") == NOT_CALLABLE) + { + not_callable.push(need(r, "Gene")?); + } + let mut low_gq = Vec::new(); + for r in rows.iter().filter(|r| stripped(r, "Quality") == LOW_GQ) { + let gene = need(r, "Gene")?; + let min = stripped(r, "Min_GQ"); + let min_gq = if pycompat::text::isdigit(min) { + opt_int(crate::int(min).ok()) + } else { + Value::Null + }; + low_gq.push(obj([ + ("gene", gene), + ("diplotype", Value::Str(stripped(r, "Diplotype").to_owned())), + ("min_gq", min_gq), + ( + "weakest", + Value::Str(stripped(r, "Weakest_variant").to_owned()), + ), + ])); + } + let cyp2d6_evidence = cyp_row.map(|r| { + [ + stripped(r, "Phenotype").to_owned(), + stripped(r, "Defining_filter").to_owned(), + stripped(r, "Quality").to_owned(), + ] + }); + let dir = text::dirname(path); + let dc = crate::join(dir, &format!("{s}_cyp2d6_depth_check.tsv")); + let depth_check = if crate::is_file(&dc) { + let mut kv: IndexMap = IndexMap::new(); + for line in text::lines(&dc).map_err(|e| e.note())? { + if let Some((k, v)) = line.split_once('\t') { + kv.insert(k.to_owned(), v.to_owned()); + } + } + Some( + DEPTH_CHECK_KEYS + .iter() + .filter_map(|k| kv.get(*k).map(|v| ((*k).to_owned(), v.clone()))) + .collect(), + ) + } else { + None + }; + let comp = crate::join(dir, &format!("{s}_pharmcat_comparison.tsv")); + let comparison = if crate::is_file(&comp) { + let c = rows_or_note(&comp)?; + let n = + |f: &dyn Fn(Option<&str>) -> bool| c.iter().filter(|r| f(cell(r, "Match"))).count(); + Some([ + n(&|m| m == Some("No")), + n(&|m| matches!(m, Some("pypgx only" | "PharmCAT only"))), + n(&|m| m == Some("Yes")), + ]) + } else { + None + }; + Ok(Pypgx { + genes_total: rows.len(), + genes_called: called, + cyp2d6: if cyp.is_empty() { "not called" } else { cyp }.to_owned(), + cnv_call, + not_callable, + low_gq, + cyp2d6_evidence, + depth_check, + comparison, + }) + } + + pub fn to_json(&self) -> Value { + let mut m: IndexMap = IndexMap::new(); + m.insert("genes_total".into(), Value::Int(self.genes_total as i128)); + m.insert("genes_called".into(), Value::Int(self.genes_called as i128)); + m.insert("cyp2d6".into(), Value::Str(self.cyp2d6.clone())); + m.insert( + "cnv_call".into(), + Value::Object( + self.cnv_call + .iter() + .map(|(k, v)| (k.clone(), Value::Str(v.clone()))) + .collect(), + ), + ); + m.insert( + "not_callable".into(), + Value::Array(self.not_callable.clone()), + ); + m.insert("low_gq".into(), Value::Array(self.low_gq.clone())); + if let Some([p, f, q]) = &self.cyp2d6_evidence { + m.insert( + "cyp2d6_evidence".into(), + obj([ + ("phenotype", Value::Str(p.clone())), + ("defining_filter", Value::Str(f.clone())), + ("quality", Value::Str(q.clone())), + ]), + ); + } + if let Some(d) = &self.depth_check { + m.insert( + "depth_check".into(), + Value::Object( + d.iter() + .map(|(k, v)| (k.clone(), Value::Str(v.clone()))) + .collect(), + ), + ); + } + if let Some([c, o, y]) = self.comparison { + m.insert( + "comparison".into(), + obj([ + ("conflicts", Value::Int(c as i128)), + ("one_tool_only", Value::Int(o as i128)), + ("concordant", Value::Int(y as i128)), + ]), + ); + } + Value::Object(m) + } +} + +// --- Cyrius -------------------------------------------------------------------------- + +#[derive(Debug, Clone, PartialEq)] +pub struct Cyrius { + pub genotype: String, + /// `None` for a row cut short of the column. + pub filter: Option, +} + +impl Cyrius { + pub fn read(path: &str) -> Result { + let rows = rows_or_note(path)?; + let r = rows.first(); + let genotype = r.and_then(|r| cell(r, "Genotype")).unwrap_or(""); + let filter = match r.map(|r| r.get("Filter")) { + None | Some(None) => Some(String::new()), + Some(Some(v)) => v.map(str::to_owned), + }; + Ok(Cyrius { + genotype: if genotype.is_empty() { + "none" + } else { + genotype + } + .to_owned(), + filter, + }) + } + + pub fn to_json(&self) -> Value { + obj([ + ("genotype", Value::Str(self.genotype.clone())), + ("filter", opt_s(self.filter.as_deref())), + ]) + } +} + +// --- the cyp2d6 block ---------------------------------------------------------------- + +/// Step 36's CYP2D6 row, as the summary keeps it. +#[derive(Debug, Clone, PartialEq)] +pub struct Consensus { + pub result: String, + pub passed_to_pharmcat: bool, + pub reason: String, + pub source: Option, +} + +/// pypgx's call alone, labelled. +#[derive(Debug, Clone, PartialEq)] +pub struct Probable { + pub diplotype: String, + pub phenotype: String, +} + +/// The `cyp2d6` block of summary.json. +#[derive(Debug, Clone, PartialEq)] +pub struct Cyp2d6 { + pub pharmcat: Value, + pub pypgx: Value, + pub cyrius: Value, + pub agree: Option, + pub consensus: Option, + pub probable: Option, +} + +/// `probable_cyp2d6`: pypgx's call when it is the only usable one. +fn probable( + diplotype: &str, + evidence: Option<&[String; 3]>, + cyrius: Option<&Cyrius>, + depth_status: Option<&str>, +) -> Option { + if !is_call(diplotype) { + return None; + } + if let Some(c) = cyrius + && is_call(&c.genotype) + && c.filter.as_deref() == Some("PASS") + { + return None; + } + if depth_status.is_some_and(|d| d != "ok") { + return None; + } + let [phenotype, filter, quality] = evidence.cloned().unwrap_or_default(); + // A row without the evidence passes None, which neither list holds. + let have = evidence.is_some(); + if !have + || !matches!(filter.as_str(), "PASS" | "none") + || !matches!(quality.as_str(), "ok" | "N/A") + { + return None; + } + Some(Probable { + diplotype: strip(diplotype, None).to_owned(), + phenotype: strip(&phenotype, None).to_owned(), + }) +} + +/// `read_consensus`: step 36's CYP2D6 row, or `None` when the table is +/// missing, older than a caller's result it was made from, or has no +/// CYP2D6 row. +fn read_consensus( + found: Option<(String, String)>, + dir: &str, + sources: [Option<&str>; 2], +) -> Result, String> { + let Some((p, rel)) = found else { + return Ok(None); + }; + let mtime = |x: &str| pycompat::mtime::mtime(x).map_err(|e| crate::os_error(&e, x)); + let own = mtime(&p)?; + for src in sources.into_iter().flatten() { + if mtime(&crate::join(dir, src))? > own { + return Ok(None); + } + } + let rows = text::rows(&p).map_err(|e| e.text())?; + let Some(row) = rows.iter().find(|r| stripped(r, "Gene") == "CYP2D6") else { + return Ok(None); + }; + Ok(Some(Consensus { + result: text::cell_or(row, "Result").to_owned(), + passed_to_pharmcat: cell(row, "Outside_call") == Some("yes"), + reason: text::cell_or(row, "Reason").to_owned(), + source: Some(rel), + })) +} + +impl Cyp2d6 { + /// `cyp2d6_block` over the three caller sections. + pub fn new( + cpic: &Section, + pypgx: &Section, + cyrius: &Section, + consensus_at: Option<(String, String)>, + dir: &str, + ) -> Cyp2d6 { + let pc = match &cpic.data { + Some(Data::Cpic(c)) => c.cyp2d6(), + _ => None, + }; + let pd = match (&pypgx.data, pypgx.state) { + (Some(Data::Pypgx(p)), State::Ok) => Some(p), + _ => None, + }; + let cy = match (&cyrius.data, cyrius.state) { + (Some(Data::Cyrius(c)), State::Ok) => Some(c), + _ => None, + }; + let failed = |s: &Section| { + (s.state == State::Failed).then(|| Value::Str(format!("failed (step {})", s.spec.step))) + }; + let pg = pd.map(|p| p.cyp2d6.as_str()); + let pg_ok = pg.is_some_and(is_call); + let cy_ok = cy.is_some_and(|c| is_call(&c.genotype) && c.filter.as_deref() == Some("PASS")); + let cy_txt = cy.map(|c| { + let mut t = c.genotype.clone(); + if is_call(&t) && !cy_ok { + let f = c + .filter + .as_deref() + .filter(|f| !f.is_empty()) + .unwrap_or("missing"); + t.push_str(&format!(" (not usable: Filter {f})")); + } + t + }); + let pharmcat = match pc { + Some(g) => { + let st = &g.status; + if matches!(st, Value::Str(s) if s == "not called" || s == "ambiguous") { + st.clone() + } else { + g.diplotype.clone() + } + } + None => failed(cpic).unwrap_or(Value::Null), + }; + let agree = match (pg_ok && cy_ok, pg, cy) { + (true, Some(a), Some(c)) => Some(same_diplotype(a, &c.genotype)), + _ => None, + }; + let consensus = match read_consensus( + consensus_at, + dir, + [pypgx.source.as_deref(), cyrius.source.as_deref()], + ) { + Ok(c) => c, + Err(e) => Some(Consensus { + result: "unreadable".into(), + passed_to_pharmcat: false, + reason: e, + source: None, + }), + }; + let probable = match (pg_ok, pd) { + (true, Some(p)) => probable( + &p.cyp2d6, + p.cyp2d6_evidence.as_ref(), + cy, + p.depth_check + .as_ref() + .and_then(|d| d.get("status")) + .map(String::as_str), + ), + _ => None, + }; + Cyp2d6 { + pharmcat, + pypgx: failed(pypgx).unwrap_or_else(|| opt_s(pg)), + cyrius: failed(cyrius).unwrap_or_else(|| opt_s(cy_txt.as_deref())), + agree, + consensus, + probable, + } + } + + pub fn to_json(&self) -> Value { + obj([ + ( + "calls", + obj([ + ("PharmCAT", self.pharmcat.clone()), + ("pypgx", self.pypgx.clone()), + ("Cyrius", self.cyrius.clone()), + ]), + ), + ("agree", self.agree.map_or(Value::Null, Value::Bool)), + ( + "consensus", + self.consensus.as_ref().map_or(Value::Null, |c| { + obj([ + ("result", Value::Str(c.result.clone())), + ("passed_to_pharmcat", Value::Bool(c.passed_to_pharmcat)), + ("reason", Value::Str(c.reason.clone())), + ("source", opt_s(c.source.as_deref())), + ]) + }), + ), + ( + "probable", + self.probable.as_ref().map_or(Value::Null, |p| { + obj([ + ("diplotype", Value::Str(p.diplotype.clone())), + ("phenotype", Value::Str(p.phenotype.clone())), + ("label", Value::Str(PROBABLE.into())), + ]) + }), + ), + ]) + } +} diff --git a/rust/crates/og-summary/src/collect/scan.rs b/rust/crates/og-summary/src/collect/scan.rs index 26d7c0087..2792c67d6 100644 --- a/rust/crates/og-summary/src/collect/scan.rs +++ b/rust/crates/og-summary/src/collect/scan.rs @@ -87,6 +87,25 @@ impl FolderIndex { } None } + + /// Every file one pattern names, in sorted order (`glob` with the + /// order made stable), for the readers that take the newest of them. + pub fn all(&self, pattern: &str) -> Vec<&str> { + if !has_magic(pattern) { + return self + .files + .get(pattern) + .map(String::as_str) + .into_iter() + .collect(); + } + let pat: Vec<&str> = pattern.split('/').collect(); + self.sorted + .iter() + .filter(|f| matches(&f.split('/').collect::>(), &pat)) + .map(String::as_str) + .collect() + } } fn has_magic(s: &str) -> bool { diff --git a/rust/crates/og-summary/src/collect/sections.rs b/rust/crates/og-summary/src/collect/sections.rs index 8aa04d534..a719a593a 100644 --- a/rust/crates/og-summary/src/collect/sections.rs +++ b/rust/crates/og-summary/src/collect/sections.rs @@ -20,6 +20,15 @@ pub enum Data { MosaicSexLoss(Mosaic), ReadCount(ReadCount), Variants(Variants), + ClinVar(super::clinvar::ClinVar), + Carrier(super::carrier::Carrier), + PharmCat(super::pgx::PharmCat), + Cpic(super::pgx::Cpic), + Pypgx(super::pgx::Pypgx), + Cyrius(super::pgx::Cyrius), + Cpsr(super::cpsr::Cpsr), + Clinical(super::clinical::Clinical), + Slivar(super::clinical::Slivar), } impl Data { @@ -32,6 +41,15 @@ impl Data { Data::MosaicSexLoss(d) => d.to_json(), Data::ReadCount(d) => d.to_json(), Data::Variants(d) => d.to_json(), + Data::ClinVar(d) => d.to_json(), + Data::Carrier(d) => d.to_json(), + Data::PharmCat(d) => d.to_json(), + Data::Cpic(d) => d.to_json(), + Data::Pypgx(d) => d.to_json(), + Data::Cyrius(d) => d.to_json(), + Data::Cpsr(d) => d.to_json(), + Data::Clinical(d) => d.to_json(), + Data::Slivar(d) => d.to_json(), } } } diff --git a/rust/crates/og-summary/src/collect/text.rs b/rust/crates/og-summary/src/collect/text.rs new file mode 100644 index 000000000..a215514b0 --- /dev/null +++ b/rust/crates/og-summary/src/collect/text.rs @@ -0,0 +1,195 @@ +//! The small readers the clinical sections (group b) share: a VCF's data +//! lines as cells, INFO and FORMAT as maps, a table whose read errors keep +//! their kind, and Python's way of reading a digit cell. + +use crate::{basename, os_error, text_lines}; +use pycompat::csv::{DictReader, DictRow, Quoting}; +use pycompat::text::{isdigit, strip}; +use std::collections::HashMap; + +/// Why a file could not be read: an `OSError` (its `str()`), or a fault of +/// another kind, already written as the section's note. +#[derive(Debug, Clone)] +pub enum ReadError { + Os(String), + Other(String), +} + +impl ReadError { + /// The note when the Python wrapped the error in `Unreadable(f"{name}: {e}")`. + pub fn unreadable(&self, path: &str) -> String { + match self { + ReadError::Os(e) => format!("{}: {e}", basename(path)), + ReadError::Other(n) => n.clone(), + } + } + + /// The note of the collector's catch-all: `type(e).__name__: e`. + pub fn note(&self) -> String { + match self { + ReadError::Os(e) => format!("{}: {e}", os_kind(e)), + ReadError::Other(n) => n.clone(), + } + } + + /// `str(e)`, as a message that embeds the error prints it. + pub fn text(&self) -> String { + match self { + ReadError::Os(e) => e.clone(), + ReadError::Other(n) => n.split_once(": ").map_or(n.clone(), |(_, m)| m.to_owned()), + } + } +} + +fn os_kind(e: &str) -> &'static str { + if e.contains("No such file") { + "FileNotFoundError" + } else if e.contains("Permission denied") { + "PermissionError" + } else if e.contains("Is a directory") { + "IsADirectoryError" + } else { + "OSError" + } +} + +/// The lines of a text file (gzip by its magic, universal newlines, bad +/// UTF-8 as U+FFFD), each without its line ending. +pub fn lines(path: &str) -> Result, ReadError> { + let it = og_io::Lines::new( + og_io::open(path).map_err(|e| ReadError::Os(os_error(&e, path)))?, + og_io::Utf8::Replace, + ); + it.map(|l| l.map_err(|e| ReadError::Os(gz_text(&e)))) + .collect() +} + +/// gzip's own words for a stream cut short; the reader's otherwise. +fn gz_text(e: &std::io::Error) -> String { + if e.kind() == std::io::ErrorKind::UnexpectedEof { + "Compressed file ended before the end-of-stream marker was reached".to_owned() + } else { + e.to_string() + } +} + +/// The data lines of a VCF split at tabs (`vcf_records`). +pub fn vcf_records(path: &str) -> Result>, ReadError> { + Ok(lines(path)? + .into_iter() + .filter(|l| !l.starts_with('#')) + .map(|l| l.split('\t').map(str::to_owned).collect()) + .collect()) +} + +/// The rows of a tab-separated table with a header (`csv.DictReader`). +pub fn rows(path: &str) -> Result, ReadError> { + let mut lines = text_lines(path).map_err(ReadError::Os)?; + let iter = std::iter::from_fn(move || { + let mut s = String::new(); + match lines.read_line(&mut s) { + Ok(true) => Some(Ok(s)), + Ok(false) => None, + Err(e) => Some(Err(e)), + } + }); + DictReader::new(iter, '\t', Quoting::Minimal) + .collect::, _>>() + .map_err(|e| match e.exception() { + "OSError" => ReadError::Os(e.to_string()), + kind => ReadError::Other(format!("{kind}: {e}")), + }) +} + +/// The header of a table as `csv.reader`'s first row reads it. +pub fn columns(path: &str) -> Result, ReadError> { + let first = lines(path)?.into_iter().next().unwrap_or_default(); + if first.is_empty() { + return Ok(Vec::new()); + } + Ok(first.split('\t').map(str::to_owned).collect()) +} + +/// `row.get(key)` where a missing key and a short row both read as `None`. +pub fn cell<'a>(row: &'a DictRow, key: &str) -> Option<&'a str> { + row.get(key).flatten() +} + +/// `row.get(key) or ""`. +pub fn cell_or<'a>(row: &'a DictRow, key: &str) -> &'a str { + cell(row, key).unwrap_or("") +} + +/// `(row.get(key) or "").strip()`. +pub fn stripped<'a>(row: &'a DictRow, key: &str) -> &'a str { + strip(cell_or(row, key), None) +} + +/// `row[key]`: a `KeyError` note when the table has no such column. A +/// short row gives `None`. +pub fn need<'a>(row: &'a DictRow, key: &str) -> Result, String> { + row.get(key) + .ok_or_else(|| format!("KeyError: {}", crate::repr(key))) +} + +/// `int(x) if x.isdigit() else None`. +pub fn digits(x: Option<&str>) -> Option { + x.filter(|v| isdigit(v)).and_then(|v| crate::int(v).ok()) +} + +/// `info_map`: a VCF INFO field as a map, a later key over an earlier one. +pub fn info_map(field: &str) -> HashMap<&str, &str> { + field + .split(';') + .map(|kv| kv.split_once('=').unwrap_or((kv, ""))) + .collect() +} + +/// `dict(zip(FORMAT.split(":"), SAMPLE.split(":")))`. +pub fn format_map<'a>(fmt: &'a str, sample: &'a str) -> HashMap<&'a str, &'a str> { + fmt.split(':').zip(sample.split(':')).collect() +} + +/// One FORMAT field of the first sample, `None` when absent, empty or `.`. +pub fn gt_field(r: &[String], key: &str) -> Option { + if r.len() < 10 { + return None; + } + format_map(&r[8], &r[9]) + .get(key) + .filter(|v| !v.is_empty() && **v != ".") + .map(|v| (*v).to_owned()) +} + +/// The GT of the first sample, `""` without one. +pub fn gt_of(r: &[String]) -> &str { + r.get(9).map_or("", |s| s.split(':').next().unwrap_or("")) +} + +/// `int(r[1])`, as the note of the collector's catch-all when it is not one. +pub fn pos(r: &[String]) -> Result { + crate::int(&r[1]).map_err(|e| format!("ValueError: {e}")) +} + +/// Python's `max()` over floats: the first of the largest, NaN kept only +/// when it comes first. +pub fn py_max(xs: impl IntoIterator) -> Option { + xs.into_iter().fold(None, |best, x| match best { + Some(b) if x.partial_cmp(&b) != Some(std::cmp::Ordering::Greater) => Some(b), + _ => Some(x), + }) +} + +/// `os.path.relpath(path, base)` for a path found under `base`. +pub fn relpath(base: &str, path: &str) -> String { + let b = base.trim_end_matches('/'); + path.strip_prefix(b) + .and_then(|r| r.strip_prefix('/')) + .unwrap_or(path) + .to_owned() +} + +/// `os.path.dirname(p)`. +pub fn dirname(p: &str) -> &str { + p.rfind('/').map_or("", |i| &p[..i]) +} diff --git a/rust/crates/og-summary/src/collect/two_hits.rs b/rust/crates/og-summary/src/collect/two_hits.rs new file mode 100644 index 000000000..52d2184b8 --- /dev/null +++ b/rust/crates/og-summary/src/collect/two_hits.rs @@ -0,0 +1,980 @@ +//! Two hits in one gene across callers: a called deletion plus a small +//! variant. +//! +//! Small variants alone miss a gene hit twice when one hit is a deletion: +//! an exon deletion of one copy of MUTYH plus a pathogenic variant on the +//! other, an HBA deletion plus an HBA point variant, one SMN1 copy plus an +//! SMN1 variant, a 1q21.1 deletion plus the RBM8A low-expression allele +//! (TAR syndrome: Albers et al., Nat Genet 2012, PMID 22366785). The +//! deletions come from the engines that call them: AnnotSV's class 4 and 5 +//! deletions (step 05), step 35's interval table, its HBA count and +//! Parascopy's SMN1 count. +//! +//! The two are in trans (biallelic) only when the reads show it: the +//! variant sits inside the deletion and is called homozygous (one copy is +//! left, and it carries the variant), or both calls carry read-backed phase +//! (a `|` GT and the same PS) on different copies. On the same copy (cis) +//! they are no second hit; otherwise they are possibly in trans. +//! +//! The readers here take from step 05's and step 35's tables only what the +//! rule needs. The AnnotSV and Parascopy sections themselves are read by +//! their own modules. + +use super::clinvar::obj; +use super::text::{self, ReadError, cell, columns, digits, rows}; +use indexmap::IndexMap; +use pycompat::csv::DictRow; +use pycompat::json::Value; +use pycompat::text::isdigit; +use std::collections::{BTreeSet, HashMap}; + +/// Gene spans: chrom, start, end (1-based, inclusive). +pub type Spans = IndexMap; + +/// A called deletion that reaches a gene. +#[derive(Debug, Clone, PartialEq)] +pub struct Deletion { + pub chrom: String, + pub start: i128, + pub end: i128, + pub text: String, + /// A variant inside the span is on the one copy left. False for the + /// HBA count, which cannot say which gene of the pair is gone. + pub inside: bool, + /// The call's GT and phase set (AnnotSV only). + pub gt: Option, + pub ps: Option, +} + +/// A small variant of a gene, as the rule reads it. +#[derive(Debug, Clone, PartialEq)] +pub struct Variant { + pub gene: String, + pub key: VKey, + pub chrom: String, + pub pos: i128, + pub text: String, + pub genotype: String, + pub copies: usize, + pub ploidy: usize, + pub gt: String, + pub ps: Option, + /// The severity name of a hypomorph allele. + pub allele: Option, +} + +/// A variant key: a small variant `(chrom, pos, ref, alt)`. +pub type VKey = (String, i128, String, String); + +/// How a deletion and a small variant in one gene pair up. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Verdict { + /// Both copies are hit. + Biallelic, + /// Phase unknown, or the calls disagree: possibly both copies. + PossiblyBiallelic, + /// Both on the same copy: no second hit. + Cis, +} + +impl Verdict { + pub fn word(self) -> &'static str { + match self { + Verdict::Biallelic => "biallelic", + Verdict::PossiblyBiallelic => "possibly biallelic", + Verdict::Cis => "cis", + } + } +} + +/// The best pair of a gene. +#[derive(Debug, Clone, PartialEq)] +pub struct Entry { + pub gene: String, + pub verdict: Verdict, + pub why: String, + pub deletion: String, + pub variant: String, + pub genotype: String, + pub allele: Option, +} + +impl Entry { + pub fn to_json(&self) -> IndexMap { + let Value::Object(m) = obj([ + ("gene", Value::Str(self.gene.clone())), + ("verdict", Value::Str(self.verdict.word().into())), + ("why", Value::Str(self.why.clone())), + ("deletion", Value::Str(self.deletion.clone())), + ("variant", Value::Str(self.variant.clone())), + ("genotype", Value::Str(self.genotype.clone())), + ( + "allele", + self.allele.clone().map_or(Value::Null, Value::Str), + ), + ]) else { + unreachable!() + }; + m + } +} + +/// The copies a phased GT puts the alt allele on, or `None` when the GT +/// is not phased or a copy is not called (the missing copy could carry the +/// other allele, so no cis or trans can be read from it). +pub fn phase_haps(gt: Option<&str>) -> Option> { + let gt = gt.filter(|g| !g.is_empty() && g.contains('|'))?; + let alleles: Vec<&str> = gt.split('|').collect(); + if alleles.contains(&".") { + return None; + } + Some( + alleles + .iter() + .enumerate() + .filter(|(_, a)| **a != "0") + .map(|(i, _)| i) + .collect(), + ) +} + +/// The verdict of one deletion and one small variant in one gene, and why. +pub fn pair_phase(dl: &Deletion, v: &Variant) -> (Verdict, String) { + let inside = dl.inside && dl.chrom == v.chrom && dl.start <= v.pos && v.pos <= dl.end; + // Read-backed phase first: on the same copy (cis) is no second hit, + // wherever the variant sits. + let hd = phase_haps(dl.gt.as_deref()).filter(|h| !h.is_empty()); + let hv = phase_haps(Some(&v.gt)).filter(|h| !h.is_empty()); + let mut phased: Option<(Verdict, String)> = None; + if let (Some(hd), Some(hv)) = (&hd, &hv) + && let Some(ps) = dl.ps.as_deref().filter(|p| !p.is_empty()) + && Some(ps) == v.ps.as_deref() + { + if hd == hv { + return ( + Verdict::Cis, + format!("in cis: read-backed phase puts both on the same copy (phase set {ps})"), + ); + } + phased = Some(if hd.is_disjoint(hv) { + ( + Verdict::Biallelic, + format!( + "in trans: read-backed phase puts them on different copies (phase set {ps})" + ), + ) + } else { + ( + Verdict::PossiblyBiallelic, + format!( + "read-backed phase puts the variant on both copies (phase set {ps}); check the reads" + ), + ) + }); + } + if inside && v.copies >= 2 { + return ( + Verdict::Biallelic, + "the variant is called homozygous inside the deletion: it is hemizygous, on the one \ + copy the deletion leaves" + .into(), + ); + } + if inside && v.ploidy == 2 { + return ( + Verdict::PossiblyBiallelic, + "a heterozygous call inside a called deletion: one of the two calls may be wrong; \ + check the reads" + .into(), + ); + } + if let Some(p) = phased { + return p; + } + ( + Verdict::PossiblyBiallelic, + "phase unknown, possibly in trans (no read-backed phase links the two calls)".into(), + ) +} + +/// What [`two_hits`] finds. +#[derive(Debug, Default)] +pub struct TwoHits { + /// The best pair of each gene hit twice, in the order genes were found. + pub entries: IndexMap, + /// Variants called homozygous inside a deletion (hemizygous): the + /// deletion's text. + pub hemi: HashMap<(String, VKey), String>, + /// The best pair of each variant that has one: ` with + /// : `. + pub pairs: HashMap<(String, VKey), String>, +} + +/// The genes hit twice. +pub fn two_hits(variants: &[Variant], dels: &HashMap>) -> TwoHits { + let mut out = TwoHits::default(); + let mut best: HashMap<(String, VKey), Verdict> = HashMap::new(); + for v in variants { + let k = (v.gene.clone(), v.key.clone()); + for dl in dels.get(&v.gene).map_or(&[][..], Vec::as_slice) { + let (verdict, why) = pair_phase(dl, v); + if verdict == Verdict::Biallelic && why.starts_with("the variant is called homozygous") + { + out.hemi.insert(k.clone(), dl.text.clone()); + } + if verdict == Verdict::Cis { + continue; + } + let e = Entry { + gene: v.gene.clone(), + verdict, + why: why.clone(), + deletion: dl.text.clone(), + variant: v.text.clone(), + genotype: if out.hemi.contains_key(&k) { + "hemizygous".into() + } else { + v.genotype.clone() + }, + allele: v.allele.clone(), + }; + match out.entries.get(&v.gene) { + Some(old) if old.verdict <= verdict => {} + _ => { + out.entries.insert(v.gene.clone(), e); + } + } + if best.get(&k).is_none_or(|b| verdict < *b) { + best.insert(k.clone(), verdict); + out.pairs.insert( + k.clone(), + format!("{} with {}: {why}", verdict.word(), dl.text), + ); + } + } + } + out +} + +// --- the deletions -------------------------------------------------------------------- + +/// One AnnotSV row as the two-hit and ACMG rules read it. +#[derive(Debug, Clone, PartialEq)] +pub struct SvRow { + /// `SV_chrom:SV_start-SV_end` as AnnotSV writes it. + pub sv: String, + pub sv_type: Option, + pub gt: Option, + pub ps: Option, + pub acmg_class: Option, + pub genes: Vec, + pub caller: Option, + /// Callers supporting it in step 22's consensus; `None` without one. + pub support: Option, + pub common: bool, +} + +impl SvRow { + /// `str(acmg_class)`. + pub fn class(&self) -> &str { + self.acmg_class.as_deref().unwrap_or("None") + } +} + +/// Step 05's checked class 4 and 5 table: its class 4 and 5 rows, `None` +/// when there is no table, or the error text the rules print. +pub fn read_annotsv( + at: Option<(String, String)>, + consensus: Option, +) -> Result>, String> { + let Some((p, _)) = at else { + return Ok(None); + }; + let size = std::fs::metadata(&p) + .map_err(|e| crate::os_error(&e, &p))? + .len(); + let rows = if size > 0 { + rows(&p).map_err(|e| e.text())? + } else { + Vec::new() + }; + const NEED: [&str; 9] = [ + "ACMG_class", + "Caller", + "Common", + "Exon_edge", + "Population_AF", + "SV_chrom", + "SV_end", + "SV_start", + "SV_type", + ]; + if let Some(r) = rows.first() + && NEED.iter().any(|k| r.get(k).is_none()) + { + let list: Vec = NEED.iter().map(|k| crate::repr(k)).collect(); + return Err(format!( + "{} lacks a column of [{}]", + crate::basename(&p), + list.join(", ") + )); + } + let support = consensus_support(&rows, consensus.as_deref()); + let py = |v: Option<&str>| v.unwrap_or("None").to_owned(); + let out = rows + .iter() + .zip(support) + .map(|(r, support)| { + let fmt = sample_format(r); + let some = |k: &str| { + fmt.get(k) + .filter(|v| !v.is_empty()) + .map(|v| (*v).to_owned()) + }; + SvRow { + sv: format!( + "{}:{}-{}", + py(cell(r, "SV_chrom")), + py(cell(r, "SV_start")), + py(cell(r, "SV_end")) + ), + sv_type: cell(r, "SV_type").map(str::to_owned), + gt: some("GT"), + ps: some("PS"), + acmg_class: cell(r, "ACMG_class").map(str::to_owned), + genes: text::cell_or(r, "Gene_name") + .split(';') + .filter(|g| !g.is_empty()) + .map(str::to_owned) + .collect(), + caller: cell(r, "Caller").map(str::to_owned), + support, + common: cell(r, "Common") == Some("yes"), + } + }) + .filter(|x| matches!(x.class(), "4" | "5")) + .collect(); + Ok(Some(out)) +} + +/// The first sample's FORMAT values of an AnnotSV row: the column after +/// FORMAT in the header, none when that is `Annotation_mode` or absent. +fn sample_format(r: &DictRow) -> HashMap<&str, &str> { + let keys: Vec<&str> = r.iter().map(|(k, _)| k).collect(); + let Some(i) = keys.iter().position(|k| *k == "FORMAT") else { + return HashMap::new(); + }; + match keys.get(i + 1) { + Some(&k) if k != "Annotation_mode" => { + text::format_map(text::cell_or(r, "FORMAT"), text::cell_or(r, k)) + } + _ => HashMap::new(), + } +} + +/// How far a consensus record may sit from an AnnotSV call, each end. +const CONSENSUS_BP: i128 = 1000; + +fn chrom_key(c: &str) -> &str { + if c.len() >= 3 && c[..3].eq_ignore_ascii_case("chr") { + &c[3..] + } else { + c + } +} + +fn info_value<'a>(info: &'a str, key: &str) -> &'a str { + info.split(';') + .find_map(|kv| { + let (k, v) = kv.split_once('=').unwrap_or((kv, "")); + (k == key).then_some(v) + }) + .unwrap_or("") +} + +fn dup_or(t: &str) -> String { + let t = t.to_uppercase(); + if t.starts_with("DUP") { + "DUP".into() + } else { + t + } +} + +/// The support of each row from step 22's consensus: the nearest record of +/// the same contig and type within 1 kb at both ends; 1 without one; `None` +/// for every row when there is no consensus or it cannot be read. +fn consensus_support(rows: &[DictRow], path: Option<&str>) -> Vec> { + let none = || vec![None; rows.len()]; + let Some(path) = path.filter(|p| crate::is_file(p)) else { + return none(); + }; + let Ok(lines) = text::lines(path) else { + return none(); + }; + // (contig, type) to the records' (pos, end, supp). + type Records = HashMap<(String, String), Vec<(i128, i128, i128)>>; + let mut by: Records = HashMap::new(); + for l in lines.iter().filter(|l| !l.starts_with('#')) { + let c: Vec<&str> = l.split('\t').collect(); + if c.len() < 8 { + continue; + } + let num = |s: &str| crate::int(s).ok(); + let Some(pos) = num(c[1]) else { continue }; + let end = match info_value(c[7], "END") { + "" => Some(pos), + e => num(e), + }; + let supp = match info_value(c[7], "SUPP") { + "" => Some(0), + e => num(e), + }; + let (Some(end), Some(supp)) = (end, supp) else { + continue; + }; + by.entry(( + chrom_key(c[0]).to_owned(), + dup_or(info_value(c[7], "SVTYPE")), + )) + .or_default() + .push((pos, end, supp)); + } + rows.iter() + .map(|r| { + let (Some(Ok(start)), Some(Ok(end))) = ( + cell(r, "SV_start").map(crate::int), + cell(r, "SV_end").map(crate::int), + ) else { + return Some(1); + }; + let key = ( + chrom_key(cell(r, "SV_chrom").unwrap_or("")).to_owned(), + dup_or(cell(r, "SV_type").unwrap_or("")), + ); + let mut best: Option<(i128, i128)> = None; + for &(p, e, supp) in by.get(&key).map_or(&[][..], Vec::as_slice) { + let (dp, de) = ((p - start).abs(), (e - end).abs()); + if dp <= CONSENSUS_BP && de <= CONSENSUS_BP && best.is_none_or(|(d, _)| dp + de < d) + { + best = Some((dp + de, supp)); + } + } + Some(best.map_or(1, |(_, s)| s)) + }) + .collect() +} + +/// What step 35 wrote that the rule reads. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct Step35 { + /// The interval table's deletions and duplications; `None` without it. + pub intervals: Option>>, + /// The HBA table (metric to value); `None` without it. + pub hba: Option>, + /// Parascopy's resolved SMN1 copy number at exon 7. + pub smn1: Option, +} + +/// SMN1 exon 7, c.840, GRCh38 1-based. +pub const SMN1_EXON7: (&str, i128) = ("chr5", 70951946); +/// The tables in paralogs/ that are not Parascopy catalog tables. +const NOT_CATALOG_TABLES: [&str; 2] = ["hba", "paraphase"]; +const PARASCOPY_COLUMNS: [&str; 9] = [ + "chrom", + "start", + "end", + "agCN", + "agCN_filter", + "agCN_qual", + "psCN", + "psCN_qual", + "psCN_filter", +]; +const INTERVAL_COLUMNS: [&str; 9] = [ + "locus", "name", "chrom", "start", "end", "status", "copies", "expected", "change", +]; +const DOSAGE_COLUMNS: [&str; 6] = ["locus", "status", "copies", "estimate", "reason", "reading"]; + +fn has_all(path: &str, cols: &[&str]) -> Result { + let have = columns(path).map_err(|e| e.text())?; + Ok(cols.iter().all(|c| have.iter().any(|h| h == c))) +} + +fn short_row(path: &str, rows: &[DictRow], cols: &[&str]) -> Result<(), String> { + if let Some(i) = rows + .iter() + .position(|r| cols.iter().any(|k| cell(r, k).is_none())) + { + return Err(format!( + "{} line {} lacks a field", + crate::basename(path), + i + 2 + )); + } + Ok(()) +} + +/// Step 35's HBA or CYP21A2 table as metric to value; a table without a +/// status row reads as not assessed. +fn read_hba(path: &str) -> Result, String> { + let mut out = IndexMap::new(); + for l in text::lines(path).map_err(|e| e.text())? { + if let Some((k, v)) = l.split_once('\t') + && k != "metric" + { + out.insert(k.to_owned(), v.to_owned()); + } + } + if !matches!( + out.get("status").map(String::as_str), + Some("called" | "not assessed") + ) { + out = IndexMap::from([ + ("status".to_owned(), "not assessed".to_owned()), + ( + "message".to_owned(), + format!("not assessed: {} has no status row", crate::basename(path)), + ), + ]); + } + Ok(out) +} + +/// The psCN of SMN1 at exon 7 when Parascopy resolved it (filter PASS, +/// quality 20 or more, a whole number), as `parascopy_cn_at` reads it. +fn smn1_cn(rows: &[DictRow]) -> Option { + let (chrom, pos) = SMN1_EXON7; + for r in rows { + let (Some(c), Some(Ok(s)), Some(Ok(e))) = ( + cell(r, "chrom"), + cell(r, "start").map(crate::int), + cell(r, "end").map(crate::int), + ) else { + continue; + }; + let mut regions = vec![(c.to_owned(), s + 1, e)]; + for h in text::cell(r, "homologous_regions") + .filter(|v| !v.is_empty()) + .unwrap_or("*") + .split(',') + { + let h = pycompat::text::strip(h, None); + if let Some((hc, rest)) = h.split_once(':') { + let a: String = rest.chars().take_while(char::is_ascii_digit).collect(); + let rest = &rest[a.len()..]; + if let Some(rest) = rest.strip_prefix('-') { + let b: String = rest.chars().take_while(char::is_ascii_digit).collect(); + if !hc.is_empty() && !a.is_empty() && !b.is_empty() { + regions.push((hc.to_owned(), crate::int(&a).ok()?, crate::int(&b).ok()?)); + } + } + } + } + let Some(idx) = regions + .iter() + .position(|(c, b, e)| c == chrom && *b <= pos && pos <= *e) + else { + continue; + }; + let cns: Vec<&str> = text::cell_or(r, "psCN").split(',').collect(); + let quals: Vec<&str> = text::cell_or(r, "psCN_qual").split(',').collect(); + let cn = cns.get(idx).copied().unwrap_or("?"); + let q = quals + .get(idx) + .and_then(|q| pycompat::num::parse_float(q).ok()); + let filt = cell(r, "psCN_filter") + .filter(|v| !v.is_empty()) + .unwrap_or("."); + let ok = filt == "PASS" && q.is_some_and(|q| q >= 20.0) && isdigit(cn); + return if ok { crate::int(cn).ok() } else { None }; + } + None +} + +/// What the rule takes from step 35's tables, in the order the Parascopy +/// section reads them, so the same fault stops it. `Ok(None)`: step 35 +/// wrote no table. +pub fn read_step35(dir: &str, s: &str, assets: &str) -> Result, String> { + let paralogs = crate::join(dir, "paralogs"); + let mut tables: Vec<(String, String)> = Vec::new(); + if let Ok(entries) = std::fs::read_dir(¶logs) { + for e in entries.flatten() { + let Ok(name) = e.file_name().into_string() else { + continue; + }; + let Some(t) = name + .strip_prefix(&format!("{s}_")) + .and_then(|n| n.strip_suffix("_copy_number.tsv")) + else { + continue; + }; + let p = crate::join(¶logs, &name); + if !NOT_CATALOG_TABLES.contains(&t) && crate::is_file(&p) { + tables.push((t.to_owned(), p)); + } + } + } + tables.sort_by(|a, b| pycompat::mtime::py_cmp(&a.1, &b.1)); + let side = |n: &str| crate::join(¶logs, &format!("{s}_{n}")); + let (hp, ip, cp, dp) = ( + side("hba_copy_number.tsv"), + side("cn_intervals.tsv"), + side("cyp21a2_psv.tsv"), + side("cn_dosage.tsv"), + ); + let smn = tables + .iter() + .find(|(t, _)| t == "smn") + .map(|(_, p)| p.clone()); + let any = + smn.is_some() || !tables.is_empty() || [&hp, &ip, &cp, &dp].iter().any(crate::is_file); + if !any { + return Ok(None); + } + let mut smn1 = None; + if let Some(p) = &smn { + let r = rows(p).map_err(|e| e.text())?; + if !has_all(p, &PARASCOPY_COLUMNS)? { + return Err(format!( + "{} lacks the chrom, start, end, agCN, agCN_filter, agCN_qual, psCN, psCN_qual or \ + psCN_filter column", + crate::basename(p) + )); + } + smn1 = smn1_cn(&r); + } + let others: Vec<&(String, String)> = tables.iter().filter(|(t, _)| t != "smn").collect(); + if !others.is_empty() { + let cat = super::assets::asset_rows(assets, "paralog_loci.tsv")?; + for r in &cat { + if r.get("table").is_none() { + return Err("'table'".into()); + } + } + } + let hba = if crate::is_file(&hp) { + Some(read_hba(&hp)?) + } else { + None + }; + let intervals = if crate::is_file(&ip) { + let r = rows(&ip).map_err(|e| e.text())?; + if !has_all(&ip, &INTERVAL_COLUMNS)? { + return Err(format!( + "{} lacks the locus, name, status, copies, expected or change column", + crate::basename(&ip) + )); + } + short_row(&ip, &r, &INTERVAL_COLUMNS)?; + Some( + r.iter() + .filter(|x| matches!(cell(x, "change"), Some("deletion" | "duplication"))) + .map(|x| { + [ + "locus", "name", "chrom", "start", "end", "copies", "expected", "change", + ] + .iter() + .map(|k| ((*k).to_owned(), text::cell_or(x, k).to_owned())) + .collect() + }) + .collect(), + ) + } else { + None + }; + if crate::is_file(&cp) { + read_hba(&cp)?; + } + if crate::is_file(&dp) { + if !has_all(&dp, &DOSAGE_COLUMNS)? { + return Err(format!( + "{} lacks the {} or reading column", + crate::basename(&dp), + DOSAGE_COLUMNS[..5].join(", ") + )); + } + let r = rows(&dp).map_err(|e| e.text())?; + short_row(&dp, &r, &DOSAGE_COLUMNS)?; + } + for (_, p) in others { + rows(p).map_err(|e: ReadError| e.text())?; + columns(p).map_err(|e| e.text())?; + } + Ok(Some(Step35 { + intervals, + hba, + smn1, + })) +} + +/// The deletion inputs of the sample, read once for every rule that needs +/// them. +#[derive(Debug, Clone)] +pub struct DeletionInputs { + pub annotsv: Result>, String>, + pub step35: Result, String>, +} + +/// The genes a called deletion reaches: the deletions of each gene, the +/// engines read, and the engines not read with why. +pub fn gene_deletions( + inp: &DeletionInputs, + spans: &Spans, +) -> (HashMap>, Vec, Vec) { + let mut out: HashMap> = HashMap::new(); + let (mut read, mut unread) = (Vec::new(), Vec::new()); + let over = |chrom: &str, a: i128, b: i128| -> Vec { + spans + .iter() + .filter(|(_, (c, gs, ge))| c == chrom && *gs <= b && *ge >= a) + .map(|(g, _)| g.clone()) + .collect() + }; + let mut add = |g: String, x: Deletion| out.entry(g).or_default().push(x); + match &inp.annotsv { + Err(e) => unread.push(format!("AnnotSV (step 05): {e}")), + Ok(None) => unread.push("AnnotSV (step 05): no table".into()), + Ok(Some(rows)) => { + read.push("AnnotSV class 4 and 5 deletions (step 05)".into()); + for x in rows { + let (chrom, span) = x.sv.split_once(':').unwrap_or((&x.sv, "")); + let (a, b) = span.split_once('-').unwrap_or((span, "")); + if x.sv_type.as_deref() != Some("DEL") || x.common || !(isdigit(a) && isdigit(b)) { + continue; + } + // A call genotyped 0/0 is no deletion in this sample. + if let Some(gt) = x.gt.as_deref() + && gt.replace('|', "/").split('/').all(|a| a == "0") + { + continue; + } + let chrom = if chrom.starts_with("chr") { + chrom.to_owned() + } else { + format!("chr{chrom}") + }; + let (Ok(a_n), Ok(b_n)) = (crate::int(a), crate::int(b)) else { + continue; + }; + // SV_start is the base before the deleted bases. + let first = a_n + 1; + let mut genes: BTreeSet = over(&chrom, first, b_n).into_iter().collect(); + genes.extend(x.genes.iter().filter(|g| !spans.contains_key(*g)).cloned()); + for g in genes { + add( + g, + Deletion { + chrom: chrom.clone(), + start: first, + end: b_n, + inside: true, + text: format!( + "DEL {chrom}:{a}-{b} (AnnotSV class {}, {})", + x.class(), + x.caller.as_deref().unwrap_or("None") + ), + gt: x.gt.clone(), + ps: x.ps.clone(), + }, + ); + } + } + } + } + let st = match &inp.step35 { + Ok(Some(st)) => st, + Ok(None) => { + unread.push("step 35's copy-number tables: none".into()); + return (out, read, unread); + } + Err(e) => { + unread.push(format!("step 35's copy-number tables: {e}")); + return (out, read, unread); + } + }; + match &st.intervals { + Some(rows) => { + read.push("step 35's interval table (read depth)".into()); + for r in rows { + let f = |k: &str| r.get(k).map_or("", String::as_str); + if f("change") != "deletion" || !(isdigit(f("start")) && isdigit(f("end"))) { + continue; + } + let (Ok(a), Ok(b)) = (crate::int(f("start")), crate::int(f("end"))) else { + continue; + }; + for g in over(f("chrom"), a + 1, b) { + add( + g, + Deletion { + chrom: f("chrom").to_owned(), + start: a + 1, + end: b, + inside: true, + gt: None, + ps: None, + text: format!( + "{} deletion, {} of {} copies (read depth, step 35)", + f("locus"), + f("copies"), + f("expected") + ), + }, + ); + } + } + } + None => unread.push("step 35's interval table: none".into()), + } + let hba = st.hba.clone().unwrap_or_default(); + let n = hba.get("alpha_globin_genes").map_or("", String::as_str); + if hba.get("status").map(String::as_str) == Some("called") && isdigit(n) { + read.push("step 35's HBA1/HBA2 count".into()); + // 0 genes: no alpha-globin gene is left to carry a variant. + if digits(Some(n)).is_some_and(|k| (1..4).contains(&k)) { + for g in ["HBA1", "HBA2"] { + if let Some((c, a, b)) = spans.get(g) { + add( + g.to_owned(), + Deletion { + chrom: c.clone(), + start: *a, + end: *b, + inside: false, + gt: None, + ps: None, + text: format!("{n} of 4 alpha-globin genes (read depth, step 35)"), + }, + ); + } + } + } + } else { + unread.push(format!( + "step 35's HBA1/HBA2 count: {}", + hba.get("message") + .filter(|m| !m.is_empty()) + .map_or("not called", String::as_str) + )); + } + match st.smn1 { + Some(cn) => { + read.push("Parascopy's SMN1 count (step 35)".into()); + if cn == 1 + && let Some((c, a, b)) = spans.get("SMN1") + { + add( + "SMN1".into(), + Deletion { + chrom: c.clone(), + start: *a, + end: *b, + inside: true, + gt: None, + ps: None, + text: "one SMN1 copy (Parascopy, step 35)".into(), + }, + ); + } + } + None => unread.push("Parascopy's SMN1 count: not resolved".into()), + } + (out, read, unread) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn del(gt: Option<&str>, ps: Option<&str>) -> Deletion { + Deletion { + chrom: "chr1".into(), + start: 100, + end: 200, + text: "DEL".into(), + inside: true, + gt: gt.map(str::to_owned), + ps: ps.map(str::to_owned), + } + } + + fn var(pos: i128, gt: &str, ps: Option<&str>) -> Variant { + let (copies, ploidy) = super::super::clinvar::allele_copies(gt); + Variant { + gene: "G".into(), + key: ("chr1".into(), pos, "A".into(), "T".into()), + chrom: "chr1".into(), + pos, + text: format!("chr1:{pos} A>T"), + genotype: "het".into(), + copies, + ploidy, + gt: gt.into(), + ps: ps.map(str::to_owned), + allele: None, + } + } + + #[test] + fn phase_reads_only_called_phased_copies() { + assert_eq!(phase_haps(Some("0|1")), Some(BTreeSet::from([1]))); + assert_eq!(phase_haps(Some("1|0")), Some(BTreeSet::from([0]))); + assert_eq!(phase_haps(Some("0/1")), None); + assert_eq!(phase_haps(Some(".|1")), None); + assert_eq!(phase_haps(None), None); + } + + #[test] + fn read_backed_phase_decides_outside_the_deletion() { + // Outside the deletion's span: phase alone decides. + let trans = pair_phase(&del(Some("0|1"), Some("7")), &var(500, "1|0", Some("7"))); + assert_eq!(trans.0, Verdict::Biallelic); + assert!(trans.1.starts_with("in trans")); + let cis = pair_phase(&del(Some("0|1"), Some("7")), &var(500, "0|1", Some("7"))); + assert_eq!(cis.0, Verdict::Cis); + // Another phase set links nothing. + let other = pair_phase(&del(Some("0|1"), Some("7")), &var(500, "1|0", Some("8"))); + assert_eq!(other.0, Verdict::PossiblyBiallelic); + assert!(other.1.starts_with("phase unknown")); + } + + #[test] + fn a_phased_cis_pair_is_no_second_hit_even_inside_the_deletion() { + let v = var(150, "0|1", Some("7")); + let cis = pair_phase(&del(Some("0|1"), Some("7")), &v); + assert_eq!(cis.0, Verdict::Cis); + let found = two_hits( + &[v], + &HashMap::from([("G".to_owned(), vec![del(Some("0|1"), Some("7"))])]), + ); + assert!(found.entries.is_empty()); + } + + #[test] + fn homozygous_inside_is_hemizygous() { + let v = var(150, "1/1", None); + let found = two_hits( + std::slice::from_ref(&v), + &HashMap::from([("G".to_owned(), vec![del(None, None)])]), + ); + let e = &found.entries["G"]; + assert_eq!(e.verdict, Verdict::Biallelic); + assert_eq!(e.genotype, "hemizygous"); + assert!(found.hemi.contains_key(&("G".to_owned(), v.key.clone()))); + } + + #[test] + fn the_best_pair_of_a_gene_wins() { + let v = var(500, "1|0", Some("7")); + let found = two_hits( + std::slice::from_ref(&v), + &HashMap::from([( + "G".to_owned(), + vec![del(None, None), del(Some("0|1"), Some("7"))], + )]), + ); + assert_eq!(found.entries["G"].verdict, Verdict::Biallelic); + assert!(found.pairs[&("G".to_owned(), v.key)].starts_with("biallelic with DEL: in trans")); + } +} From 075b43becf59b7d339ad1eaef2afa3f9704ff232 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:57:01 +0100 Subject: [PATCH 05/17] perf(summary): read the clinical VCF and the ClinVar hits line by line The clinical VCF, slivar's VCF and the hits were read whole before being split; now one pass streams each, the hits are no longer copied for the ClinVar section, and the summary's own data is freed before the JSON text is made. A table cut short reads as Python's EOFError. Bead: pgp-bs9u.42 --- rust/crates/og-summary/src/collect.rs | 12 +- rust/crates/og-summary/src/collect/acmg.rs | 144 ++++++++++++------ .../crates/og-summary/src/collect/clinical.rs | 4 +- rust/crates/og-summary/src/collect/clinvar.rs | 35 ++--- rust/crates/og-summary/src/collect/text.rs | 49 ++++-- 5 files changed, 169 insertions(+), 75 deletions(-) diff --git a/rust/crates/og-summary/src/collect.rs b/rust/crates/og-summary/src/collect.rs index cc5e827f3..15a29828f 100644 --- a/rust/crates/og-summary/src/collect.rs +++ b/rust/crates/og-summary/src/collect.rs @@ -7,7 +7,9 @@ //! switches to it. Group `a` is the model and the first seven sections: //! coverage, callability, sex_check, sample_qc, mosaic_sex_loss, //! read_count and variants, with the run status, the manifest and each -//! section's state. +//! section's state. Group `b` is the clinical sections ([`group_b`]): +//! clinvar, carrier, pharmcat, cpic, pypgx, cyrius, cpsr, clinical (with +//! the ACMG SF tier and the two-hit rule) and slivar, and the CYP2D6 block. //! //! How it reads a folder: one walk of the folder ([`scan::FolderIndex`]) //! instead of a `glob` per candidate path; each input read once (the @@ -93,8 +95,13 @@ pub fn cli(argv: &[String], out: &mut impl Write, err: &mut impl Write) -> u8 { Ok(s) => s, Err(e) => return crate::finish(Err(e), err), }; + // The sections' own data is freed before the text is made, and the + // tree before the file is written: a large ClinVar list is in memory in + // one form at a time. + let value = summary.to_json(); + drop(summary); let body = match dumps( - &summary.to_json(), + &value, &Dump { indent: Some(1), ..Dump::default() @@ -103,6 +110,7 @@ pub fn cli(argv: &[String], out: &mut impl Write, err: &mut impl Write) -> u8 { Ok(b) => b + "\n", Err(e) => return crate::finish(Err(e.to_string()), err), }; + drop(value); let r = crate::write_file(&target, body.as_bytes()); if r.is_ok() { let _ = writeln!(out, "Summary written: {target}"); diff --git a/rust/crates/og-summary/src/collect/acmg.rs b/rust/crates/og-summary/src/collect/acmg.rs index 70439bbd8..db3077299 100644 --- a/rust/crates/og-summary/src/collect/acmg.rs +++ b/rust/crates/og-summary/src/collect/acmg.rs @@ -262,72 +262,124 @@ pub struct ClinicalVcf { } impl ClinicalVcf { + /// One pass over the file, line by line: a whole-genome clinical VCF + /// is never held in memory. pub fn read(path: &str) -> Result { - let lines = text::lines(path).map_err(|e| e.unreadable(path))?; - let records = lines.iter().filter(|l| !l.starts_with('#')).count(); + let mut records = 0; + let mut scan = HighScan::default(); + text::each_line(path, |l| { + if !l.starts_with('#') { + records += 1; + } + scan.line(l); + }) + .map_err(|e| e.unreadable(path))?; Ok(ClinicalVcf { records, - high: high_impact(&lines), + high: scan.finish(), }) } } -/// `csq_high_impact_genes`. -fn high_impact(lines: &[String]) -> Result, String> { - let floor = ClinicalFloor::read(lines)?; - let mut fmt: Option> = None; - let mut out = IndexMap::new(); - for line in lines { +/// `csq_high_impact_genes`, fed one line at a time. The floors come from +/// the header (`clinical_call_floor`), which ends at the first line that +/// does not start with `##`. +#[derive(Default)] +struct HighScan { + header: Vec, + floor: Option>, + fmt: Option>, + out: IndexMap, + err: Option, +} + +impl HighScan { + fn line(&mut self, line: &str) { + if self.floor.is_none() { + if line.starts_with("##") { + self.header.push(line.to_owned()); + } else { + self.floor = Some(ClinicalFloor::read(&std::mem::take(&mut self.header))); + } + } + if self.err.is_some() { + return; + } if line.starts_with("##INFO= = line.split('\t').map(str::to_owned).collect(); - if r.len() < 8 { - continue; + return; } - let (cls, entries, freq) = csq_rank(fm, info_map(&r[7]).get("CSQ").copied().unwrap_or("")); - let mut genes: IndexMap = IndexMap::new(); - for e in entries.into_iter().filter(|e| e.impact == "HIGH") { - genes.entry(e.gene).or_insert(e.consequence); + if let Err(e) = high_record(fm, line, floor, &mut self.out) { + self.err = Some(e); } - if genes.is_empty() { - continue; + } + + fn finish(mut self) -> Result, String> { + let floor = self + .floor + .take() + .unwrap_or_else(|| ClinicalFloor::read(&self.header)); + floor?; + match self.err { + Some(e) => Err(e), + None => Ok(self.out), } - let gt = text::gt_of(&r).to_owned(); - let (weak, why, gq, dp, ad) = if r.len() > 9 { - call_check(&r[8], &r[9], &floor) - } else { - (false, Vec::new(), None, None, None) - }; - let pos = text::pos(&r)?; - out.insert( - (r[0].clone(), pos, r[3].clone(), r[4].clone()), - HighRecord { - genotype: gt, - ps: gt_field(&r, "PS"), - genes, - transcript: cls, - frequency: freq, - weak, - why, - gq, - dp, - ad, - }, - ); } - Ok(out) +} + +/// One data line of `csq_high_impact_genes`. +fn high_record( + fm: &[String], + line: &str, + floor: &ClinicalFloor, + out: &mut IndexMap, +) -> Result<(), String> { + let r: Vec = line.split('\t').map(str::to_owned).collect(); + if r.len() < 8 { + return Ok(()); + } + let (cls, entries, freq) = csq_rank(fm, info_map(&r[7]).get("CSQ").copied().unwrap_or("")); + let mut genes: IndexMap = IndexMap::new(); + for e in entries.into_iter().filter(|e| e.impact == "HIGH") { + genes.entry(e.gene).or_insert(e.consequence); + } + if genes.is_empty() { + return Ok(()); + } + let gt = text::gt_of(&r).to_owned(); + let (weak, why, gq, dp, ad) = if r.len() > 9 { + call_check(&r[8], &r[9], floor) + } else { + (false, Vec::new(), None, None, None) + }; + let pos = text::pos(&r)?; + out.insert( + (r[0].clone(), pos, r[3].clone(), r[4].clone()), + HighRecord { + genotype: gt, + ps: gt_field(&r, "PS"), + genes, + transcript: cls, + frequency: freq, + weak, + why, + gq, + dp, + ad, + }, + ); + Ok(()) } /// A source of a candidate variant. diff --git a/rust/crates/og-summary/src/collect/clinical.rs b/rust/crates/og-summary/src/collect/clinical.rs index 6dc161381..5936475e6 100644 --- a/rust/crates/og-summary/src/collect/clinical.rs +++ b/rust/crates/og-summary/src/collect/clinical.rs @@ -107,7 +107,9 @@ impl Slivar { let Some(v) = vcf else { return Ok(Slivar(Value::Null)); }; - let n = text::vcf_records(v).map_err(|e| e.unreadable(v))?.len(); + let mut n = 0; + text::each_line(v, |l| n += usize::from(!l.starts_with('#'))) + .map_err(|e| e.unreadable(v))?; m.insert("prioritized".into(), int(n)); comphet_phase( &crate::join(text::dirname(v), &format!("{s}_compound_het_pairs.tsv")), diff --git a/rust/crates/og-summary/src/collect/clinvar.rs b/rust/crates/og-summary/src/collect/clinvar.rs index 7350c5a6b..f930c25e7 100644 --- a/rust/crates/og-summary/src/collect/clinvar.rs +++ b/rust/crates/og-summary/src/collect/clinvar.rs @@ -79,22 +79,20 @@ pub struct HitsVcf { impl HitsVcf { pub fn read(path: String, rel: String) -> HitsVcf { - let read = text::lines(&path) - .map(|lines| { - let mut floor = Floor::default(); - for l in lines.iter().take_while(|l| l.starts_with("##")) { - if l.starts_with("##PGP_CALL_FLOOR=") { - floor_values(l, false, &mut floor); - } - } - let recs = lines - .iter() - .filter(|l| !l.starts_with('#')) - .map(|l| l.split('\t').map(str::to_owned).collect()) - .collect(); - (floor, recs) - }) - .map_err(|e: ReadError| e.unreadable(&path)); + let mut floor = Floor::default(); + let mut header = true; + let mut recs: Vec> = Vec::new(); + let read = text::each_line(&path, |l| { + header &= l.starts_with("##"); + if header && l.starts_with("##PGP_CALL_FLOOR=") { + floor_values(l, false, &mut floor); + } + if !l.starts_with('#') { + recs.push(l.split('\t').map(str::to_owned).collect()); + } + }) + .map(|()| (floor, recs)) + .map_err(|e: ReadError| e.unreadable(&path)); HitsVcf { path, rel, read } } } @@ -622,7 +620,10 @@ impl ClinVar { side: &dyn Fn(&str) -> Option<(String, String)>, s: &str, ) -> Result { - let (floor, recs) = hits.read.clone()?; + let (floor, recs) = match &hits.read { + Ok((f, r)) => (*f, r), + Err(e) => return Err(e.clone()), + }; let mut rows = Vec::new(); for r in recs.iter().filter(|r| r.len() >= 8) { rows.push(Hit::parse(r, floor)?); diff --git a/rust/crates/og-summary/src/collect/text.rs b/rust/crates/og-summary/src/collect/text.rs index a215514b0..d9f554d65 100644 --- a/rust/crates/og-summary/src/collect/text.rs +++ b/rust/crates/og-summary/src/collect/text.rs @@ -42,7 +42,9 @@ impl ReadError { } fn os_kind(e: &str) -> &'static str { - if e.contains("No such file") { + if e == GZ_EOF { + "EOFError" + } else if e.contains("No such file") { "FileNotFoundError" } else if e.contains("Permission denied") { "PermissionError" @@ -64,22 +66,29 @@ pub fn lines(path: &str) -> Result, ReadError> { .collect() } +/// Python's gzip on a stream cut short (an `EOFError`). +const GZ_EOF: &str = "Compressed file ended before the end-of-stream marker was reached"; + /// gzip's own words for a stream cut short; the reader's otherwise. fn gz_text(e: &std::io::Error) -> String { if e.kind() == std::io::ErrorKind::UnexpectedEof { - "Compressed file ended before the end-of-stream marker was reached".to_owned() + GZ_EOF.to_owned() } else { e.to_string() } } -/// The data lines of a VCF split at tabs (`vcf_records`). -pub fn vcf_records(path: &str) -> Result>, ReadError> { - Ok(lines(path)? - .into_iter() - .filter(|l| !l.starts_with('#')) - .map(|l| l.split('\t').map(str::to_owned).collect()) - .collect()) +/// Calls `f` with each line of a text file in turn (as [`lines`] reads +/// them), never holding the whole file. +pub fn each_line(path: &str, mut f: impl FnMut(&str)) -> Result<(), ReadError> { + let it = og_io::Lines::new( + og_io::open(path).map_err(|e| ReadError::Os(os_error(&e, path)))?, + og_io::Utf8::Replace, + ); + for l in it { + f(&l.map_err(|e| ReadError::Os(gz_text(&e)))?); + } + Ok(()) } /// The rows of a tab-separated table with a header (`csv.DictReader`). @@ -90,6 +99,9 @@ pub fn rows(path: &str) -> Result, ReadError> { match lines.read_line(&mut s) { Ok(true) => Some(Ok(s)), Ok(false) => None, + Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => { + Some(Err(std::io::Error::new(e.kind(), gz_text(&e)))) + } Err(e) => Some(Err(e)), } }); @@ -193,3 +205,22 @@ pub fn relpath(base: &str, path: &str) -> String { pub fn dirname(p: &str) -> &str { p.rfind('/').map_or("", |i| &p[..i]) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::testutil::Dir; + + #[test] + fn a_table_cut_short_is_python_s_eof_error() { + let d = Dir::new("text-cut"); + let body = "A\tB\n".to_owned() + &"1\t2\n".repeat(2000); + let full = std::fs::read(d.put_gz("t.tsv.gz", &body)).unwrap(); + for n in [30, full.len() / 2, full.len() - 4] { + let p = d.put("cut.tsv.gz", &full[..n]); + let e = rows(&p).unwrap_err(); + assert_eq!(e.unreadable(&p), format!("cut.tsv.gz: {GZ_EOF}"), "{n}"); + assert_eq!(e.note(), format!("EOFError: {GZ_EOF}"), "{n}"); + } + } +} From 391581c8be60770e4d0a42fd6a7a2d4dab8459b4 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:57:01 +0100 Subject: [PATCH 06/17] fix(summary): word a broken PharmCAT report as Python's json does The note now says "line L column C (char N)" in code points after universal newlines, and names a UTF-8 BOM, as json.load does. Bead: pgp-bs9u.42 --- rust/crates/og-summary/src/collect/pgx.rs | 50 ++++++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/rust/crates/og-summary/src/collect/pgx.rs b/rust/crates/og-summary/src/collect/pgx.rs index 458660493..d59db94c2 100644 --- a/rust/crates/og-summary/src/collect/pgx.rs +++ b/rust/crates/og-summary/src/collect/pgx.rs @@ -50,14 +50,37 @@ pub struct PharmCat { pub html_report: bool, } +/// `json.load` of a file opened as text: universal newlines, and Python's +/// `JSONDecodeError` text (`MSG: line L column C (char N)`, in code points). +fn json_load(raw: &str) -> Result { + let text = raw.replace("\r\n", "\n").replace('\r', "\n"); + if text.starts_with('\u{feff}') { + return Err( + "Unexpected UTF-8 BOM (decode using utf-8-sig): line 1 column 1 (char 0)".into(), + ); + } + pycompat::json::loads(&text).map_err(|e| match e { + pycompat::json::JsonError::Decode(what, at) => { + let before = &text[..at.min(text.len())]; + let pos = before.chars().count(); + let line = before.matches('\n').count() + 1; + let col = match before.rfind('\n') { + Some(i) => before[i + 1..].chars().count() + 1, + None => pos + 1, + }; + format!("{what}: line {line} column {col} (char {pos})") + } + other => other.to_string(), + }) +} + impl PharmCat { pub fn read(path: &str) -> Result { let raw = std::fs::read(path) .map_err(|e| format!("{}: {}", crate::basename(path), crate::os_error(&e, path)))?; let body = String::from_utf8(raw) .map_err(|e| format!("{}: 'utf-8' codec can't decode: {e}", crate::basename(path)))?; - let data = - pycompat::json::loads(&body).map_err(|e| format!("{}: {e}", crate::basename(path)))?; + let data = json_load(&body).map_err(|e| format!("{}: {e}", crate::basename(path)))?; let version = match &data { Value::Object(m) => match m.get("pharmcatVersion") { None => String::new(), @@ -616,3 +639,26 @@ impl Cyp2d6 { ]) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn json_errors_read_as_python_words_them() { + assert_eq!( + json_load("{\"pharmcatVersion\": ").unwrap_err(), + "Expecting value: line 1 column 21 (char 20)" + ); + assert_eq!( + json_load("{\r\n\"é\": x").unwrap_err(), + "Expecting value: line 2 column 6 (char 7)" + ); + assert!( + json_load("\u{feff}{}") + .unwrap_err() + .starts_with("Unexpected UTF-8 BOM") + ); + assert!(json_load("[1]").is_ok()); + } +} From 4c06a83e5bd339e2c41e14589954ba715f261415 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:57:01 +0100 Subject: [PATCH 07/17] test(parity): the clinical sections of summary collect against the Python Ten cases of og summary collect --only b: every input in both layouts, failed and stale steps, unreadable inputs, a VEP output with and without its index, phased deletions in trans and in cis, and no assets. The oracle stages the helper modules the collector imports from the pinned commit instead of stubs. cb-notgz holds the one deliberate fix. Bead: pgp-bs9u.42 --- tests/parity/cases.tsv | 10 + tests/parity/gen/summary-clinical.sh | 247 ++++++++++++++++++ .../summary-collect/cb-notgz/cb-notgz.json | 180 +++++++++++++ .../inputs/summary-collect/cb_vep.vcf.gz | Bin 0 -> 727 bytes .../inputs/summary-collect/cb_vep.vcf.gz.tbi | Bin 0 -> 253 bytes tests/parity/inputs/summary-collect/oracle.sh | 11 +- 6 files changed, 444 insertions(+), 4 deletions(-) create mode 100644 tests/parity/gen/summary-clinical.sh create mode 100644 tests/parity/golden/summary-collect/cb-notgz/cb-notgz.json create mode 100644 tests/parity/inputs/summary-collect/cb_vep.vcf.gz create mode 100644 tests/parity/inputs/summary-collect/cb_vep.vcf.gz.tbi diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index 0a9afd6bf..03bf5e088 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -838,6 +838,16 @@ summary-collect empty gen:summary-collect.sh,summary-collect/oracle.sh --sample summary-collect declared-arg gen:summary-collect.sh,summary-collect/oracle.sh --sample S --sample-dir declared-status --assets assets --declared-sex female --out declared-arg.json --only a declared-arg.json json:paths=schema_version,sample,run,databases,manifest,sections.coverage,sections.callability,sections.sex_check,sections.sample_qc,sections.mosaic_sex_loss,sections.read_count,sections.variants PYTHON_IMAGE summary-collect no-assets gen:summary-collect.sh,summary-collect/oracle.sh --sample S --sample-dir full --assets nowhere --out no-assets.json --only a no-assets.json json:paths=schema_version,sample,run,databases,manifest,sections.coverage,sections.callability,sections.sex_check,sections.sample_qc,sections.mosaic_sex_loss,sections.read_count,sections.variants PYTHON_IMAGE summary-collect no-folder gen:summary-collect.sh,summary-collect/oracle.sh --sample S --sample-dir no-such-folder --assets assets --out no-folder.json --only a no-folder.json json:paths=schema_version,sample,run,databases,manifest,sections.coverage,sections.callability,sections.sex_check,sections.sample_qc,sections.mosaic_sex_loss,sections.read_count,sections.variants PYTHON_IMAGE +summary-collect cb-full gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-full --assets assets --out cb-full.json --only b cb-full.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-nf gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-nf --assets assets --out cb-nf.json --only b cb-nf.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-probable gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-probable --assets assets --out cb-probable.json --only b cb-probable.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-failed gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-failed --assets assets --out cb-failed.json --only b cb-failed.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-bad gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-bad --assets assets --out cb-bad.json --only b cb-bad.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-nfbad gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-nfbad --assets assets --out cb-nfbad.json --only b cb-nfbad.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-notbi gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-notbi --assets assets --out cb-notbi.json --only b cb-notbi.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-notgz gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-notgz --assets assets --out cb-notgz.json --only b cb-notgz.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-empty gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-empty --assets assets --out cb-empty.json --only b cb-empty.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-no-assets gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-full --assets nowhere --out cb-no-assets.json --only b cb-no-assets.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE y-tree-positions small gen:y-inputs.sh --tree-dir t pos.txt pos.txt bytes PYTHON_IMAGE y-tree-positions big gen:y-inputs.sh --tree-dir ./big/ pos.txt pos.txt bytes PYTHON_IMAGE y-tree-positions odd gen:y-inputs.sh --tree-dir odd pos.txt pos.txt bytes PYTHON_IMAGE diff --git a/tests/parity/gen/summary-clinical.sh b/tests/parity/gen/summary-clinical.sh new file mode 100644 index 000000000..aa1191f44 --- /dev/null +++ b/tests/parity/gen/summary-clinical.sh @@ -0,0 +1,247 @@ +#!/usr/bin/env bash +# summary-clinical.sh: writes the sample folders of the summary-collect +# parity cases of group b (og summary collect --only b) in the current +# directory, the same every time. The clinical sections: ClinVar hits with +# each review status, a call floor, combined alleles, half calls and +# phased calls, the cDNA check, scan and deep intronic tables, gnomAD AF +# from an indexed VEP output; the carrier list; PharmCAT, CPIC, pypgx and +# Cyrius and step 36's CYP2D6 row; CPSR with its own class beside +# ClinVar's; the clinical filter's table and VCF with the ACMG SF tier; +# slivar's table and VCF with the phased compound-het pairs; and the +# deletions the two-hit rule pairs with a variant (AnnotSV, step 35's +# interval, HBA and SMN1 tables), in trans, in cis and inside the deletion. +# +# The VEP output is the committed pair cb_vep.vcf.gz and its .tbi, staged +# beside the folders; vep/S_vep.vcf.gz links to it. assets/ is a copy of +# the repository's gene lists. pymods/ holds the helper modules the Python +# collector imports, at a pinned commit, so the oracle reads them instead +# of stubs. scripts/ci/oracle-diff.sh runs it. +set -euo pipefail +export LC_ALL=C TZ=UTC +here=$(cd "$(dirname "$0")" && pwd) +root=$(cd "${here}/../../.." && pwd) + +put() { mkdir -p "$(dirname "$1")"; printf '%b' "$2" > "$1"; } +putgz() { mkdir -p "$(dirname "$1")"; printf '%b' "$2" | gzip -n -c > "$1"; } +at() { touch -t "$1" "${@:2}"; } # [[CC]YY]MMDDhhmm[.SS], in UTC + +T0=1577836800 # 2020-01-01 00:00:00 UTC, the start of the run +OLD=201906011230.00 # before the run +NEW=202103040506.07 # after it +NEWER=202103040507.00 + +# The helper modules of bin/collect_summary.py, as they were when the +# collector was last the summary's writer. +HELPERS=7c1470bc50933f48a1a0dde3c0e0b2b87bc48718 +git -C "$root" cat-file -e "${HELPERS}^{commit}" 2> /dev/null \ + || git -C "$root" fetch -q --depth=1 origin "$HELPERS" +mkdir -p pymods +for m in pgx_outside_calls clip_check annotsv_postfilter paraphase_report traits_report; do + git -C "$root" show "${HELPERS}:bin/${m}.py" > "pymods/${m}.py" +done + +mkdir -p assets +for a in acmg_sf.tsv carrier_genes.tsv paralog_loci.tsv; do + cp "${root}/assets/${a}" assets/ +done + +VCF_HEAD='##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n' +PL='criteria_provided,_multiple_submitters,_no_conflicts' + +# --- the ClinVar hits of step 06 --------------------------------------------------- +HITS="##fileformat=VCFv4.2\n##PGP_CALL_FLOOR=GQ=20,DP=8\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" +HITS+="chr1\t45331556\t1\tC\tT\t.\tPASS\tGENEINFO=MUTYH:4595;CLNSIG=Pathogenic;CLNREVSTAT=${PL}\tGT:GQ:DP:PS\t0|1:50:30:45331000\n" +HITS+="chr6\t26092913\t2\tG\tA\t.\tPASS\tGENEINFO=HFE:3077;CLNSIG=Pathogenic;CLNREVSTAT=reviewed_by_expert_panel\tGT:GQ:DP\t0/1:99:40\n" +HITS+="chr7\t117559590\t3\tATCT\tA\t.\tPASS\tGENEINFO=CFTR:1080;CLNSIG=Pathogenic;CLNREVSTAT=practice_guideline\tGT:GQ:DP\t0/1:15:30\n" +HITS+="chr13\t20189473\t4\tC\tT\t.\tPASS\tGENEINFO=GJB2:2706;CLNSIG=Likely_pathogenic;CLNREVSTAT=criteria_provided,_single_submitter;PGP_JOINED_FROM=chr13:20189473:C:T|chr13:20189475:G:A\tGT:GQ:DP\t1/.:40:5\n" +HITS+="chr3\t15645186\t5\tG\tC\t.\tPASS\tGENEINFO=BTD:686;CLNSIG=Pathogenic/Likely_pathogenic;CLNREVSTAT=criteria_provided,_conflicting_classifications\tGT\t0/1\n" +HITS+="chr17\t43057100\t6\tC\tT\t.\tPASS\tGENEINFO=BRCA1:672|NBR2:10230;CLNSIG=Pathogenic;CLNREVSTAT=reviewed_by_expert_panel;PGP_PART_OF=chr17:43057100:C:T|chr17:43057102:A:G\tGT:GQ:DP\t0/1:60:35\n" +HITS+="chrX\t101398000\t7\tC\tT\t.\tPASS\tGENEINFO=GLA:2717;CLNSIG=Pathogenic;CLNREVSTAT=no_assertion_criteria_provided;PGP_ONE_ALLELE=chrX:101398004:G:A\tGT\t1\n" +HITS+="chr16\t176700\t8\tG\tA\t.\tPASS\tGENEINFO=HBA1:3039;CLNSIG=Pathogenic;CLNREVSTAT=${PL}\tGT:GQ:DP\t0/1:45:31\n" +HITS+="chr5\t70951946\t9\tC\tT\t.\tPASS\tGENEINFO=SMN1:6606;CLNSIG=Pathogenic;CLNREVSTAT=${PL}\tGT:GQ:DP\t1/1:45:31\n" +HITS+="chr2\t100\t10\tA\tG\t.\tPASS\tCLNREVSTAT=;CLNSIG=\tGT:DP\t0/1:x\n" +HITS+="chr2\t200\t11\tA\tG\t.\tPASS\n" + +CDNA='chrom\tpos\tref\talt\talt_reads\talt_clipped\tref_reads\tref_clipped\tcdna_check\tcdna_note\tconstruct_site\n' +CDNA+='chr1\t45331556\tC\tT\t12\t6\t10\t0\tflagged\tpossible cDNA/transgene contamination: check the reads\tyes\n' +CDNA+='chr6\t26092913\tG\tA\t20\t0\t20\t0\tclean\t\t\n' +CDNA+='chr6\tx\tG\tA\t1\t1\t1\t1\tclean\t\t\n' +SCAN='# cdna_scan: 2 genes with spliced reads\nchrom\tstart\tend\tjunctions\treads\tjunction_list\nchr1\t100\t200\t2\t5\tA:1-2,A:3-4\nchr2\tx\t1\t1\t1\t\n' +DEEP='chrom\tpos\tref\talt\tgene\thgvs\tclinvar_id\tcall\tdetail\nchr7\t117639961\tC\tT\tCFTR\tc.3718-2477C>T\t7129\t\t\nchr7\tNA\tC\tT\tCFTR\t\t\t\t\n' + +# --- pharmacogenomics ---------------------------------------------------------------- +CPIC='Gene\tDiplotype\tPhenotype\tStatus\nCYP2D6\t*1/*4\tIntermediate Metabolizer\tnon-normal\nCYP2C19\t*1/*17\tRapid\tambiguous\nCYP2C9\t*1/*1\tNormal\tnormal\nDPYD\n' +REC='CPIC lookup\n WARNING: CYP2C19 has two possible phenotypes\nnothing\nWARNING:last\n' +PYPGX='Gene\tDiplotype\tPhenotype\tStatus\tSource\tCNV_call\tQuality\tMin_GQ\tWeakest_variant\tDefining_filter\n' +PYPGX+='CYP2D6\t*1/*2\t Normal Metabolizer \tok\tBAM\tNormal\tok\t\t\tPASS\n' +PYPGX+='CYP2A6\tFAILED\t\tfailed\tBAM\t\t\t\t\t\n' +PYPGX+='CYP2B6\t*1/*6\tIM\tdepth check failed\tbam\tN/A\tok\t\t\t\n' +PYPGX+='CYP2E1\t.\t\tnot callable on this reference\tBAM\t.\t\t\t\t\n' +PYPGX+='UGT1A1\tIndeterminate\t\tok\tVCF\t\tlow-confidence: check the reads\t 12 \tchr2:1:A:G\t\n' +PYPGX+='SLCO1B1\t*1/*5\t\tok\tBAM\t.\tlow-confidence: check the reads\tx\t\t\n' +DEPTH='metric\tvalue\nstatus\tok\ngene_mapq1_share\t0.99\nbody_mapq20_share\t0.95\nother\t1\n' +COMP='Gene\tMatch\nCYP2D6\tYes\nCYP2C19\tNo\nCYP3A5\tpypgx only\nTPMT\tPharmCAT only\n' +CONS='Gene\tResult\tOutside_call\tReason\tEvidence\nHLA-A\tx\tno\t\t\nCYP2D6\t*1/*2\tyes\tpypgx and Cyrius agree\tpypgx *1/*2; Cyrius *2/*1\n' + +# --- CPSR (step 17) -------------------------------------------------------------------- +CPSR='VAR_ID\tGENOMIC_CHANGE\tSYMBOL\tCLASSIFICATION\tCPSR_CLASSIFICATION\tCLINVAR_CLASSIFICATION\tASSERTION_AUTHORITY\tACMG_CODE\tDP_CONTROL\n' +CPSR+='17_43057100_C_T\t17:g.43057100C>T\tBRCA1\tPathogenic\tLikely_Benign\tPathogenic\tClinVar\tPVS1\t6\n' +CPSR+='\t13:g.32338000G>A\tBRCA2\tLikely_Pathogenic\tVUS\t\tCPSR\t\t40\n' +CPSR+='2_47403000_A_G\t\tMSH2\tVUS\tLP\tB\t\t\t\n' +CPSR+='x\tbad\tATM\tBenign\tBenign\t\t\t\t3\n' +CPSR+='5_112839000_A_T\t\tAPC\t\tPathogenic\t\t\t\t30\n' +CPSR_CDNA="${CDNA}17\t43057100\tC\tT\t5\t5\t9\t0\tflagged\tsplice\t\n" + +# --- the clinical filter (step 23) ------------------------------------------------------ +CSQF='Allele|Consequence|IMPACT|SYMBOL|Feature|CANONICAL|MANE_SELECT|gnomADe_AF|gnomADg_AF' +CLIN="##fileformat=VCFv4.2\n##PGP_CALL_FLOOR=GQ=20,DP=8,HET_AB=0.25-0.75,HOM_AB=0.85\n" +CLIN+="##INFO=\n" +CLIN+='#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n' +CLIN+='chr1\t45332000\t.\tG\tA\t50\tPASS\tCSQ=A|stop_gained|HIGH|MUTYH|T1|YES|NM_1|0.0001|\tGT:GQ:DP:AD:PS\t1|0:40:30:15,15:45331000\n' +CLIN+='chr2\t47414000\t.\tA\tAT\t50\tPASS\tCSQ=AT|frameshift_variant|HIGH|MSH2|T2|YES||0.00002|0.00001\tGT:GQ:DP:AD\t0/1:50:30:28,2\n' +CLIN+='chr10\t100\t.\tC\tT\t50\tPASS\tCSQ=T|stop_gained|HIGH|PTEN|T3|YES||0.2|\tGT\t0/1\n' +CLIN+='chr6\t26093000\t.\tC\tT\t50\tPASS\tCSQ=T|stop_gained|HIGH|HFE|T4|YES||&|\tGT:GQ:DP:AD\t1/1:12:5:3,4\n' +CLIN+='chr17\t43057100\t.\tC\tT\t50\tPASS\tCSQ=T|missense_variant|MODERATE|BRCA1|T5|YES|NM_2||,T|stop_gained|HIGH|BRCA1|T6|||0.0001|\tGT\t0/1\n' +CLIN+='chr11\t47332000\t.\tG\tA\t50\tPASS\tCSQ=A|splice_donor_variant&intron_variant|HIGH|MYBPC3|T7|||0.001|,A|stop_gained|HIGH|MYBPC3|T8|||0.0001|\tGT:GQ\t0/1:.\n' +CLINT='GENE\tIMPACT\tFREQ_TIER\tCALL_CHECK\nMUTYH\tHIGH\trare\tok\nMSH2\tHIGH\trare\tweak\nHFE\tHIGH\tunknown\tweak\n.\tMODERATE\trare\tok\n\t\t\t\n' + +# --- slivar (step 31) ------------------------------------------------------------------- +SLIV='CHROM\tPOS\tGENE\tFREQ_TIER\nchr1\t1\tA\trare\nchr1\t2\tA\tunknown\n' +CHETS='GENE\tCHROM\nA\tchr1\nA\tchr1\nB\tchr2\n\tchr3\n' +PAIRS='GENE\tCHROM\tPOS_1\tREF_1\tALT_1\tPOS_2\tREF_2\tALT_2\tPHASE\tTRANS_READS\tCIS_READS\n' +for i in $(seq 1 23); do PAIRS+="G${i}\tchr1\t${i}0\tA\tG\t${i}5\tC\tT\ttrans\t${i}\t1\n"; done +PAIRS+='H\tchr2\t1\tA\tG\t2\tC\tT\tcis\t0\t4\nK\tchr2\t3\tA\tG\nL\tchr2\t4\tA\tG\t5\tC\tT\tunknown\tx\t\n' + +# --- the deletions of the two-hit rule -------------------------------------------------- +ANN='SV_chrom\tSV_start\tSV_end\tSV_length\tSV_type\tGene_name\tACMG_class\tCaller\tExon_edge\tExon_edge_detail\tPopulation_AF\tPopulation_source\tCommon\tAnnotSV_class\tFORMAT\tS\tAnnotation_mode\n' +ANN+='1\t45338000\t45340000\t-2000\tDEL\tMUTYH\t5\tManta\tyes\t\t\t\tno\t5\tGT:PS\t1|0:45331000\tfull\n' +ANN+='chr2\t47386000\t47388000\t-2000\tDEL\tEPCAM\t4\tDelly\tno\t\t\t\tno\t4\tGT\t0/1\tfull\n' +ANN+='17\t43000000\t43100000\t100000\tDUP\tBRCA1;NBR2\t4\tCNVpytor\tno\t\t0.001\tgnomAD\tno\t4\tGT\t0/1\tfull\n' +ANN+='13\t20180000\t20200000\t-20000\tDEL\tGJB2;NOTAGENE\t5\tManta\tno\t\t\t\tno\t5\tGT\t0/0\tfull\n' +ANN+='13\t20180000\t20200000\t-20000\tDEL\tGJB2;XYZ1\t4\tManta\tno\t\t\t\tno\t4\tGT\t0/1\tfull\n' +ANN+='3\t15600000\t15700000\t-100000\tDEL\tBTD\t3\tManta\tno\t\t\t\tno\t5\tGT\t0/1\tfull\n' +ANN+='3\t15600000\t15700000\t-100000\tDEL\tBTD\t5\tManta\tno\t\t0.2\tgnomAD\tyes\t5\tGT\t0/1\tfull\n' +CONSV='##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tmanta\tdelly\tcnvpytor\n' +CONSV+='chr1\t45338010\t.\tN\t\t.\tPASS\tSVTYPE=DEL;END=45340020;SUPP=2;SUPP_VEC=110\n' +CONSV+='chr1\t45338500\t.\tN\t\t.\tPASS\tSVTYPE=DEL;END=45340900;SUPP=3;SUPP_VEC=111\n' +CONSV+='chr17\t43000000\t.\tN\t\t.\tPASS\tSVTYPE=DUP:TANDEM;END=43100000;SUPP=1;SUPP_VEC=0011\n' +SMN='chrom\tstart\tend\tagCN\tagCN_filter\tagCN_qual\tpsCN\tpsCN_qual\tpsCN_filter\thomologous_regions\n' +SMN+='chr5\t70925000\t70953000\t4\tPASS\t40\t1,3\t30,30\tPASS\tchr5:70049000-70078000\n' +HBA='metric\tvalue\nstatus\tcalled\nalpha_globin_genes\t3\n' +IV='locus\tname\tchrom\tstart\tend\tstatus\tcopies\texpected\tchange\n' +IV+='CFTR_del\tCFTR region\tchr7\t117500000\t117600000\tcalled\t1\t2\tdeletion\n' +IV+='DUP1\tx\tchr1\t1\t100\tcalled\t3\t2\tduplication\n' +IV+='GJB6\tx\tchr13\tx\t2\tcalled\t1\t2\tdeletion\n' + +# full: every input, the bash layout, a run whose steps were all ok. +full() { + local d=$1 + put "$d/clinvar/S_clinvar_hits.vcf" "$HITS" + put "$d/clinvar/S_clinvar_cdna_check.tsv" "$CDNA" + put "$d/clinvar/S_clinvar_cdna_scan.tsv" "$SCAN" + put "$d/clinvar/S_clinvar_deep_intronic.tsv" "$DEEP" + mkdir -p "$d/vep" + ln -s ../../cb_vep.vcf.gz "$d/vep/S_vep.vcf.gz" + ln -s ../../cb_vep.vcf.gz.tbi "$d/vep/S_vep.vcf.gz.tbi" + put "$d/pharmcat/S.report.json" '{"pharmcatVersion": "3.4.0", "genes": {}}' + put "$d/pharmcat/S.report.html" '' + put "$d/cpic/S_phenotypes.tsv" "$CPIC" + put "$d/cpic/S_cpic_recommendations.txt" "$REC" + put "$d/pypgx/S_pypgx_summary.tsv" "$PYPGX" + put "$d/pypgx/S_cyp2d6_depth_check.tsv" "$DEPTH" + put "$d/pypgx/S_pharmcat_comparison.tsv" "$COMP" + put "$d/cyrius/S_cyp2d6.tsv" 'Sample\tGenotype\tFilter\nS\t*2/*1\tPASS\n' + put "$d/pgx_consensus/S_pgx_consensus.tsv" "$CONS" + put "$d/cpsr/S.cpsr.grch38.html" '' + putgz "$d/cpsr/S.cpsr.grch38.classification.tsv.gz" "$CPSR" + put "$d/cpsr/S_cpsr_cdna_check.tsv" "$CPSR_CDNA" + put "$d/clinical/S_clinical_summary.tsv" "$CLINT" + putgz "$d/clinical/S_clinical.vcf.gz" "$CLIN" + put "$d/slivar/S_slivar_summary.tsv" "$SLIV" + put "$d/slivar/S_compound_hets.tsv" "$CHETS" + put "$d/slivar/S_compound_het_pairs.tsv" "$PAIRS" + put "$d/annotsv/S_sv_class4_5_checked.tsv" "$ANN" + putgz "$d/sv_merged/S_sv_consensus.vcf.gz" "$CONSV" + put "$d/paralogs/S_smn_copy_number.tsv" "$SMN" + put "$d/paralogs/S_hba_copy_number.tsv" "$HBA" + put "$d/paralogs/S_cn_intervals.tsv" "$IV" + put "$d/logs/run_status.tsv" "meta\tstarted_epoch\t${T0}\nstep\t06\tok\nstep\t07\tok\nstep\t17\tok\nstep\t21\tok\nstep\t23\tok\nstep\t27\tok\nstep\t31\tok\nstep\t32\tok\n" +} +full cb-full + +# nf: the Nextflow names. The clinical VCF and slivar's VCF without their +# tables, two PharmCAT reports (the newer wins), Cyrius under another name, +# CPSR by its wildcard names, no VEP output, a consensus older than pypgx, +# PharmCAT's step failed, CPIC from an earlier run, pypgx failed. +put cb-nf/clinvar/S_clinvar_hits.vcf "$HITS" +putgz cb-nf/clinical/S_clinical.vcf.gz "$CLIN" +putgz cb-nf/slivar/S_prioritized.vcf.gz "${VCF_HEAD}chr1\t1\t.\tA\tG\t1\tPASS\t.\tGT\t0/1\nchr1\t2\t.\tA\tG\t1\tPASS\t.\tGT\t0/1\n" +put cb-nf/slivar/S_compound_het_pairs.tsv "$PAIRS" +put cb-nf/vcf/A.report.json '{"pharmcatVersion": 3.2}' +put cb-nf/vcf/B.report.json '[1, 2]' +put cb-nf/cpic/S_phenotypes.tsv 'Gene\tDiplotype\tPhenotype\tStatus\nCYP2D6\t*4/*4\tPoor\tnot called\n' +put cb-nf/cyrius/T_cyp2d6.tsv 'Sample\tGenotype\tFilter\nT\t*1/*2\tLowQ\n' +put cb-nf/pypgx/S_pypgx_summary.tsv "$PYPGX" +put cb-nf/pgx_consensus/S_pgx_consensus.tsv "$CONS" +put cb-nf/cpsr/T.cpsr.grch37.html '' +putgz cb-nf/cpsr/T.cpsr.grch37.classification.tsv.gz 'SYMBOL\tFINAL_CLASSIFICATION\tCPSR_CLASSIFICATION\nA\tPathogenic\tBenign\nB\t\t\n' +put cb-nf/annotsv/S_sv_class4_5_checked.tsv '' +put cb-nf/paralogs/S_cn_intervals.tsv "$IV" +put cb-nf/logs/run_status.tsv "meta\tstarted_epoch\t${T0}\nstep\t07\tfailed (exit 1)\nstep\t27\tskipped\nstep\t32\tok\nstep\t21\tok\n" + +# probable: pypgx alone, Cyrius not usable, the depth check ok; no consensus. +put cb-probable/pypgx/S_pypgx_summary.tsv "$PYPGX" +put cb-probable/pypgx/S_cyp2d6_depth_check.tsv "$DEPTH" +put cb-probable/cyrius/S_cyp2d6.tsv 'Sample\tGenotype\tFilter\nS\t*1/*3\tLowQ\n' +put cb-probable/cpic/S_phenotypes.tsv 'Gene\tDiplotype\tPhenotype\tStatus\nCYP2C9\t*1/*1\tNormal\tnormal\n' +put cb-probable/cpic/S_cpic_recommendations.txt 'PARSING FAILED for one gene\n' + +# failed: CPIC, pypgx and Cyrius failed in the run and wrote nothing. +put cb-failed/logs/run_status.tsv "meta\tstarted_utc\t2020-02-02 02:02 UTC\nstep\t21\tfailed (exit 1)\nstep\t27\tfailed\nstep\t32\tfailed (exit 3)\nstep\t06\tok\n" +put cb-failed/pgx_consensus/S_pgx_consensus.tsv 'Gene\tResult\nCYP2C19\tx\n' + +# bad: a hit whose POS is no number, a report that is no JSON, a CPSR +# table cut short, pypgx without a Gene column, an AnnotSV table without +# its columns, a Parascopy table without its columns, a clinical floor that +# is no number, slivar's VCF cut short. +put cb-bad/clinvar/S_clinvar_hits.vcf "${VCF_HEAD}chr1\tx\t.\tA\tG\t.\tPASS\tGENEINFO=MUTYH:1;CLNSIG=Pathogenic\tGT\t0/1\n" +put cb-bad/pharmcat/S.report.json '{"pharmcatVersion": ' +putgz cb-bad/cpsr/S.cpsr.grch38.classification.tsv.gz "$CPSR" +head -c 60 cb-bad/cpsr/S.cpsr.grch38.classification.tsv.gz > cb-bad.tmp && mv cb-bad.tmp cb-bad/cpsr/S.cpsr.grch38.classification.tsv.gz +put cb-bad/pypgx/S_pypgx_summary.tsv 'Name\tSource\tCNV_call\nCYP2D6\tBAM\tNormal\n' +put cb-bad/annotsv/S_sv_class4_5_checked.tsv 'SV_chrom\tSV_start\n1\t1\n' +put cb-bad/paralogs/S_smn_copy_number.tsv 'chrom\tstart\tend\nchr5\t1\t2\n' +putgz cb-bad/clinical/S_clinical.vcf.gz "##PGP_CALL_FLOOR=GQ=20,HOM_AB=0..8\n#CHROM\n" +put cb-bad/clinical/S_clinical_summary.tsv 'GENE\tIMPACT\nA\tHIGH\n' +put cb-bad/cyrius/S_cyp2d6.tsv '' +put cb-bad/cpic/S_phenotypes.tsv 'Gene\tDiplotype\n' +putgz cb-bad/slivar/S_prioritized.vcf.gz "$CLIN" +head -c 100 cb-bad/slivar/S_prioritized.vcf.gz > cb-bad.tmp && mv cb-bad.tmp cb-bad/slivar/S_prioritized.vcf.gz + +# nf-bad: the Nextflow clinical VCF cut short; Parascopy's catalog tables +# without their catalog; an HBA table without a status. +put cb-nfbad/clinvar/S_clinvar_hits.vcf "$HITS" +putgz cb-nfbad/clinical/S_clinical.vcf.gz "$CLIN" +head -c 100 cb-nfbad/clinical/S_clinical.vcf.gz > cb-nfbad.tmp && mv cb-nfbad.tmp cb-nfbad/clinical/S_clinical.vcf.gz +put cb-nfbad/paralogs/S_hba_copy_number.tsv 'metric\tvalue\nalpha_globin_genes\t2\n' +put cb-nfbad/paralogs/S_rhd_copy_number.tsv "$SMN" + +# notbi: the VEP output without its index; the HBA count of no gene left. +put cb-notbi/clinvar/S_clinvar_hits.vcf "$HITS" +mkdir -p cb-notbi/vep +ln -s ../../cb_vep.vcf.gz cb-notbi/vep/S_vep.vcf.gz +put cb-notbi/paralogs/S_hba_copy_number.tsv 'metric\tvalue\nstatus\tcalled\nalpha_globin_genes\t0\n' + +# notgz: the clinical and slivar VCFs as plain text under a .gz name. The +# Python tells gzip by the name and finds both unreadable; og tells it by +# the first bytes and reads them (a golden). +put cb-notgz/clinical/S_clinical.vcf.gz "$CLIN" +put cb-notgz/slivar/S_prioritized.vcf.gz "${VCF_HEAD}chr1\t1\t.\tA\tG\t1\tPASS\t.\tGT\t0/1\n" + +mkdir -p cb-empty + +# Every file after the run started; the CPIC table of nf from before it; +# nf's second PharmCAT report and its consensus' sources newer. +find . -type f -exec touch -t "$NEW" {} + +at "$OLD" cb-nf/cpic/S_phenotypes.tsv cb-nf/pgx_consensus/S_pgx_consensus.tsv +at "$NEWER" cb-nf/vcf/B.report.json diff --git a/tests/parity/golden/summary-collect/cb-notgz/cb-notgz.json b/tests/parity/golden/summary-collect/cb-notgz/cb-notgz.json new file mode 100644 index 000000000..e972fd826 --- /dev/null +++ b/tests/parity/golden/summary-collect/cb-notgz/cb-notgz.json @@ -0,0 +1,180 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-13 11:46 UTC", + "run": { + "status_file": false, + "started_utc": null, + "declared_sex": null + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "clinvar": { + "title": "ClinVar screen", + "step": "06", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "carrier": { + "title": "Carrier list", + "step": "06", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "pharmcat": { + "title": "PharmCAT", + "step": "07", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cpic": { + "title": "CPIC drug-gene recommendations", + "step": "27", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "pypgx": { + "title": "pypgx", + "step": "32", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cyrius": { + "title": "Cyrius (CYP2D6)", + "step": "21", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cpsr": { + "title": "Cancer predisposition (CPSR)", + "step": "17", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "clinical": { + "title": "Clinical variant filter", + "step": "23", + "state": "ok", + "source": "clinical/S_clinical.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "variants": 6, + "acmg_sf": { + "version": "ACMG SF v3.3", + "genes_on_list": 84, + "rules": "assets/acmg_sf.tsv", + "clinvar_hits": [], + "clinvar_checked": false, + "high_impact": [ + { + "gene": "MYBPC3", + "variant": "chr11:47332000 G>A", + "genotype": "het", + "consequence": "splice_donor_variant,intron_variant", + "transcript": "ANY", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": false, + "call_note": null, + "gq": null, + "dp": null, + "ad": null, + "rule": "All P and LP", + "rule_note": null, + "severity": null + }, + { + "gene": "MSH2", + "variant": "chr2:47414000 A>AT", + "genotype": "het", + "consequence": "frameshift_variant", + "transcript": "CANONICAL", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": true, + "call_note": "alt reads 0.07 of the depth on a heterozygote", + "gq": 50, + "dp": 30, + "ad": "28,2", + "rule": "All P and LP", + "rule_note": "A deletion of the 3' end of EPCAM silences MSH2 and counts as an MSH2 finding.", + "severity": null + } + ], + "high_impact_checked": true, + "high_impact_rare": 1, + "high_impact_weak": 1, + "high_impact_frequency_unknown": 0, + "structural": [], + "structural_checked": false, + "structural_note": null, + "not_reported": [ + { + "gene": "MUTYH", + "variant": "chr1:45332000 G>A", + "genotype": "het", + "source": "HIGH impact", + "reason": "one heterozygous variant; MUTYH is reported with P and LP (2 variants)" + }, + { + "gene": "HFE", + "variant": "chr6:26093000 C>T", + "genotype": "hom", + "source": "HIGH impact", + "reason": "HFE is reported only for: HFE p.C282Y homozygotes only" + } + ], + "two_hit_note": null + } + } + }, + "slivar": { + "title": "Variant prioritization (slivar)", + "step": "31", + "state": "ok", + "source": "slivar/S_prioritized.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "prioritized": 1 + } + } + }, + "cyp2d6": { + "calls": { + "PharmCAT": null, + "pypgx": null, + "Cyrius": null + }, + "agree": null, + "consensus": null, + "probable": null + } +} diff --git a/tests/parity/inputs/summary-collect/cb_vep.vcf.gz b/tests/parity/inputs/summary-collect/cb_vep.vcf.gz new file mode 100644 index 0000000000000000000000000000000000000000..4a42ab23694fa66e2234dfd493e5ff6a313c223f GIT binary patch literal 727 zcmb2|=3rp}f&Xj_PR>jWMU0^n?q)SR2(+$0zFeR+hV`NZx8~LjU)2SZOm=ZT(V9_` zEMDRpE*v$Zef_e=W#)&^=scg3A9me;PttO?Yp&1p_I|K3RqI-p-+bljx@k`{{tB!; zbfmVk=KAUDkFPGZGk7VL{Hr!C@1CdLsg37vEq?O6&-~EUJ#Jgxd_Ht!;>uP5-*BGa zVYVlig*E?MYZsCwpRnSg?zc$Q(4$&y{7yZOrc{W0j{P%DXQPO(Z_PIKhkJF~&3-(} zy?*@t`qn4g1>S~U-0M2+$OJV@7rPWkjug+ty^7u(i&c4zB+fM&oJ?|34DXY_)U-1B z_O!<~jf`>u1wJ00^PKC~dvAO@>+?RH9SQGubUfqkF)!KfaM|GO$+ffiPF<@rlC3t| z>>f8Mf32abp4H8cc(aR%bAAL$u-=UAvY){FYuor1E&5yoTe`#^{ z1VO1gv95}4%oAHJW<+IZ*?-82>Uqe>z#xwnLE0~tcy=w29)EVmUd%rHL z{+9RZt3p1AJ9oneFbW9_xqF~bTvs0^X++XZb#Y85o#t+hDToN};+_dt;-bN5A6oYL4X#_X&m@ pEpwI$ob%A*MnLd{&Swh3ykB^CGTf+pGw(JQS4lH4gAy_r001f^JyQSx literal 0 HcmV?d00001 diff --git a/tests/parity/inputs/summary-collect/cb_vep.vcf.gz.tbi b/tests/parity/inputs/summary-collect/cb_vep.vcf.gz.tbi new file mode 100644 index 0000000000000000000000000000000000000000..1036d9b09dfc8623d118587ec9212741f883e91b GIT binary patch literal 253 zcmb2|=3oE;rvGp67;+tQ5MaF!%c=COtn3k^OF+)sH+vg3*PK;-d*s%s*)PET-bKifzuy@WGUt}pHRfL!g_g4i^_I~za ecTAsGt;;>_Vz@gV1mBI_B3_wuHv3>dBLe_=G*kHi literal 0 HcmV?d00001 diff --git a/tests/parity/inputs/summary-collect/oracle.sh b/tests/parity/inputs/summary-collect/oracle.sh index 09d629c43..1a3e15a7c 100644 --- a/tests/parity/inputs/summary-collect/oracle.sh +++ b/tests/parity/inputs/summary-collect/oracle.sh @@ -5,13 +5,16 @@ # --assets becomes ASSETS_DIR, the global render_report.py --assets sets; # --only is og's alone (the Python writes every section, and the case # compares the group's subtrees). The modules collect_summary imports are -# stubbed: none is reached for a sample folder that holds only the -# sections the cases cover, except run_delta's two readers of history/ and -# recheck/, which find nothing here. +# stubbed, except those a generator staged in pymods/ (the clinical cases +# need the real helpers); no other one is reached for a sample folder that +# holds only the sections the cases cover, except run_delta's two readers +# of history/ and recheck/, which find nothing here. prog=$1 shift d=$(mktemp -d) -sed -n 's/^import \([a-z_]*\) # noqa.*/\1/p' "$prog" | while read -r m; do : > "$d/$m.py"; done +sed -n 's/^import \([a-z_]*\) # noqa.*/\1/p' "$prog" | while read -r m; do + if [ -f "pymods/$m.py" ]; then cp "pymods/$m.py" "$d/"; else : > "$d/$m.py"; fi +done printf 'def read_recheck(path):\n return None\n\n\ndef newest_history(sample_dir):\n return None\n' > "$d/run_delta.py" cp "$prog" "$d/collect_summary.py" PYTHONPATH=$d exec python3 -c ' From f3177f5cbd50121ef60c5d1a44303481d6cb629e Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:57:01 +0100 Subject: [PATCH 08/17] docs(summary): og summary collect --only b in the changelog and the crate map Bead: pgp-bs9u.42 --- CHANGELOG.md | 4 ++++ docs/rust.md | 3 ++- rust/README.md | 2 +- rust/og/src/main.rs | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e21d1ac8..f31e9387e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The Rust `og` binary reads the clinical sections of summary.json: `og summary collect --only b`. That covers the ClinVar screen (stars, call floor, gnomAD frequencies and rsIDs from step 13's indexed VEP output, the cDNA check, scan and deep intronic tables), the carrier list, PharmCAT, CPIC, pypgx, Cyrius, CPSR, the clinical variant filter with its ACMG SF v3.3 tier and the two-hit rule, slivar with its phased compound-het pairs, and the CYP2D6 block. Nothing calls it yet: the reports still read the summary that `bin/collect_summary.py` writes. The sections are the same as the Python's on 9 parity cases, compared as JSON with key order; a tenth holds the deliberate fix below. Each input is read once: the ClinVar hits feed four sections, and the clinical VCF is read line by line in one pass for its record count and its HIGH-impact records. On a synthetic folder with 300 ClinVar hits, a 200,000-record clinical VCF and a 200,000-row slivar table, it takes 0.3 s and 51 MB, where the Python collector took 1.3 s and 135 MB. With 50,000 ClinVar hits (far more than a genome has) it takes 0.6 s but 460 MB, where the Python took 3.5 s and 113 MB, because og builds the whole summary as a JSON tree before it writes it. What differs from the Python: + - A clinical or slivar VCF named `.vcf.gz` that is plain text is read. The Python marked the section unreadable. + - A PharmCAT report that is not UTF-8 is still unreadable, but the note names the fault in Rust's words. + - The Rust `og` binary has a first part of the summary collector: `og summary collect --only a`. It reads seven sections of summary.json (coverage, callability, sex_check, sample_qc, mosaic_sex_loss, read_count and variants), the run status, the manifest and each section's state. Nothing calls it yet: the reports still read the summary that `bin/collect_summary.py` writes. The sections are the same as the Python's on 33 parity cases, compared as JSON with key order. It walks the sample folder once instead of running a `glob` for each candidate path, and it reads step 33's table once for its two sections. On a synthetic 1,000,000-record VCF the variant count takes 0.05 s and 7 MB, where the Python collector took 0.9 s and 38 MB. What differs from the Python: - A file named `.vcf.gz` that is plain text is read. The Python marked the variants section unreadable. - A sample name with `*`, `?` or `[` is matched as written. The Python read it as a file pattern. diff --git a/docs/rust.md b/docs/rust.md index cb6924559..ab177fdee 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -22,7 +22,8 @@ rust/ crates/og-setup/ what setup.sh runs once: the STRchive catalog, zips, the genetic map crates/og-steps/ the step helpers: constraint-join, prs format, step 40's consumer export; og family (step 38); og pgx outside-calls|hla-concordance|pypgx-summary (steps 08, 32, 36); og paraphase (step 35b) crates/og-summary/ the step tables of steps 20, 25 and 33: og mito, og prs table, og qc sample; - og summary collect --only a, the first seven summary sections (no caller yet) + og summary collect --only a, the first seven summary sections (no caller yet); + --only b, the clinical sections: ClinVar, carrier, PGx, CPSR, ACMG SF, two hits crates/og-pedigree/ the pedigree rules: families files, members, kinship, PARs crates/og-chip/ step 49, clonal haematopoiesis: og chip prepare, report, check-table crates/og-cn/ copy number from depth (steps 32, 35, 41, 46): og cn bed, call, intervals, check-bam, marker-regions, marker-read diff --git a/rust/README.md b/rust/README.md index 394e83e10..c42b40b64 100644 --- a/rust/README.md +++ b/rust/README.md @@ -10,7 +10,7 @@ OwnGenome's own tools, ported from `bin/*.py` one at a time into one static bina | `crates/og-bam` | BAM reading as pysam does it (fetch, `reference_end`, `cigarstring`, `get_blocks`), on htslib linked statically, and the tools that read BAMs: the reads at a variant (`reads`: one CIGAR walk per read, SA pieces, filters per tool), `og cdna check`, `og cdna scan`, `og comphet phase`, `og bam contigs`, the CYP2D6 depth check and the RFC1 motif count | | `crates/og-pedigree` | the pedigree rules of step 38: families files, members, kinship, declared relationships, PARs | | `crates/og-qc` | `og qc sex-loss`: step 33's mosaic loss of Y or X, with CPython's medians, `fmean` and exact `variance` | -| `crates/og-summary` | the step tables: `og qc sample` (step 33), `og prs table` (step 25), `og mito annotate`, `og mito deletions`, `og mito copies` (step 20); the first sections of summary.json, `og summary collect --only a`, beside the Python collector, which still writes it | +| `crates/og-summary` | the step tables: `og qc sample` (step 33), `og prs table` (step 25), `og mito annotate`, `og mito deletions`, `og mito copies` (step 20); the sections of summary.json, `og summary collect --only a` (QC) and `--only b` (ClinVar, carrier list, PGx, CPSR, the clinical filter with ACMG SF and two hits, slivar), beside the Python collector, which still writes it | | `crates/og-chip` | step 49, clonal haematopoiesis: `og chip prepare`, `og chip report`, `og chip check-table` | | `crates/og-cn` | copy number from depth, steps 32, 35, 41 and 46: `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions`, `og cn marker-read` | | `crates/og-bloodtype` | step 41, blood groups: `og bloodtype report`, `og bloodtype check-markers`, `og bloodtype systems` | diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs index 234ea7cac..119fd5441 100644 --- a/rust/og/src/main.rs +++ b/rust/og/src/main.rs @@ -50,7 +50,7 @@ const USAGE: &str = "usage: og [args] og prs format [args] og prs table [args] og mito annotate|deletions|copies [args] - og summary collect --only a [args] + og summary collect --only a|b [args] og chip prepare|report|check-table [args] og cn bed|call|intervals|check-bam|marker-regions|marker-read [args] og bloodtype report|check-markers|systems [args] From 8fb444c6c5671b44f3049406c2689da548a0e877 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:57:12 +0100 Subject: [PATCH 09/17] test(summary): red control acmg-biallelic-ignores-phase (reverted next) Plant: the deletion's read-backed phase is ignored, so a variant in cis with a deletion counts as a second hit and a trans pair is only possibly biallelic. The two_hits unit tests and the cb-full parity case must fail. Bead: pgp-bs9u.42 --- rust/crates/og-summary/src/collect/two_hits.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/crates/og-summary/src/collect/two_hits.rs b/rust/crates/og-summary/src/collect/two_hits.rs index 52d2184b8..d9c9c475d 100644 --- a/rust/crates/og-summary/src/collect/two_hits.rs +++ b/rust/crates/og-summary/src/collect/two_hits.rs @@ -143,7 +143,7 @@ pub fn pair_phase(dl: &Deletion, v: &Variant) -> (Verdict, String) { let inside = dl.inside && dl.chrom == v.chrom && dl.start <= v.pos && v.pos <= dl.end; // Read-backed phase first: on the same copy (cis) is no second hit, // wherever the variant sits. - let hd = phase_haps(dl.gt.as_deref()).filter(|h| !h.is_empty()); + let hd = phase_haps(dl.gt.as_deref()).filter(|h| h.is_empty()); let hv = phase_haps(Some(&v.gt)).filter(|h| !h.is_empty()); let mut phased: Option<(Verdict, String)> = None; if let (Some(hd), Some(hv)) = (&hd, &hv) From 1b8998fff3c1710a5194bf6b8bbee9e58fc0113b Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:59:37 +0100 Subject: [PATCH 10/17] Revert "test(dev): plant three wrong builder rules to show the tests go red" The plant turned five builder tests red in Guard's rust job (run 43417543846); back to the right rules. Bead: pgp-bs9u.35 --- rust/crates/og-dev/src/build/callability.rs | 3 +-- rust/crates/og-dev/src/build/cn_intervals.rs | 3 +-- rust/crates/og-dev/src/build/sv_rules.rs | 5 ++--- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/rust/crates/og-dev/src/build/callability.rs b/rust/crates/og-dev/src/build/callability.rs index 556c65865..d7ce4dece 100644 --- a/rust/crates/og-dev/src/build/callability.rs +++ b/rust/crates/og-dev/src/build/callability.rs @@ -111,8 +111,7 @@ fn read_gtf(path: &Path, alt: &HashMap) -> Result Vec { // Ceiling divisions of positive numbers. let n = (len + PIECE - 1) / PIECE; let step = (len + n - 1) / n; - // PLANTED, reverted in the next commit: each piece ends one base short. - out.extend((0..n).map(|i| (s + i * step, e.min(s + (i + 1) * step) - 1))); + out.extend((0..n).map(|i| (s + i * step, e.min(s + (i + 1) * step)))); } out } diff --git a/rust/crates/og-dev/src/build/sv_rules.rs b/rust/crates/og-dev/src/build/sv_rules.rs index 1007fc81b..7004f02b7 100644 --- a/rust/crates/og-dev/src/build/sv_rules.rs +++ b/rust/crates/og-dev/src/build/sv_rules.rs @@ -99,10 +99,9 @@ fn read_gtf(path: &Path) -> Result { b"gene" => { let (start, end) = r.span()?; if let Some(h) = r.attr("hgnc_id")?.filter(|h| !h.is_empty()) - && !r.attr("gene_id")?.is_some_and(|id| id.ends_with("_PAR_X")) + && !r.attr("gene_id")?.is_some_and(|id| id.ends_with("_PAR_Y")) { - // PLANTED, reverted in the next commit: the last record wins. - g.span.insert(h.to_owned(), (chrom, start, end)); + g.span.entry(h.to_owned()).or_insert((chrom, start, end)); } let (name, ty) = ( r.attr("gene_name")?.unwrap_or(""), From cebf6b944a8c14b5d8deac6cfd689a63b24b212e Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:02:06 +0100 Subject: [PATCH 11/17] fix(ci): a fetch of its own per run, and a stalled transfer counts as failed fetch_pinned downloads to a mktemp file beside the target, so two runs on one cache never write the same partial file, and gives curl a 30 s connect timeout and a 1 KB/s for 60 s floor, so a stalled server reaches the retry or the error instead of holding the build. Bead: pgp-bs9u.35 --- CHANGELOG.md | 2 +- scripts/ci/lib/fetch-pinned.sh | 36 ++++++++++++++++++++-------------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ce7f1507..20ebc7ff0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased - The three builders of the committed asset tables now run in the Rust `og` binary: `og dev build cn-intervals` (the interval rows of `bin/cn_loci.tsv`), `og dev build callability` (`assets/callability_genes.tsv`) and `og dev build sv-rule-regions` (`assets/sv_rule_regions.tsv`). They replace `scripts/ci/build-cn-intervals.py`, which is gone (the script is now `scripts/ci/build-cn-intervals.sh`), and the Python inside `scripts/ci/build-callability.sh` and `scripts/ci/build-sv-rule-regions.sh`. The scripts still download the pinned files, and the three tables are the same, byte for byte, as the committed ones. With the downloads in the cache, the callability table takes 2.5 s and 27 MB instead of 5.5 s and 39 MB, and the SV rule table 4.9 s and 30 MB instead of 10.5 s and 217 MB. What changes on purpose: - - The three scripts share one download function, `scripts/ci/lib/fetch-pinned.sh`. A file the cache folder already holds with the right sha256 is read where it is, not copied. A 403 or 404 fails at once; any other failure is tried for about five minutes, 30 s apart. `build-cn-intervals.py` tried three times, 30 s apart. + - The three scripts share one download function, `scripts/ci/lib/fetch-pinned.sh`. A file the cache folder already holds with the right sha256 is read where it is, not copied. A 403 or 404 fails at once; any other failure, a stalled transfer included, is tried for about five minutes, 30 s apart. `build-cn-intervals.py` tried three times, 30 s apart. - The marker line of `bin/cn_loci.tsv` names `scripts/ci/build-cn-intervals.sh`. No row changes. - `build-cn-intervals.sh --check` counts the lines only in the file and the lines only in the rebuild. The Python counted only the lines the rebuild lacks, so a catalog with an added line said "0 line(s) differ". - The SV rule builder takes each gene's span from the first GENCODE record of its HGNC id and skips the `_PAR_Y` copies, as the callability builder does. The Python kept the last record, so a pseudogene in a pseudoautosomal region would have taken its chrY span. No gene of the pinned files is one, so the table does not change. diff --git a/scripts/ci/lib/fetch-pinned.sh b/scripts/ci/lib/fetch-pinned.sh index ccdb82f01..f05b1c724 100644 --- a/scripts/ci/lib/fetch-pinned.sh +++ b/scripts/ci/lib/fetch-pinned.sh @@ -7,12 +7,15 @@ # Makes DIR/NAME the file at URL with that sha256. A file already there # with the right sha256 is kept, so DIR can be a cache that outlives the # run. Otherwise it prints "Downloading URL" on stderr (the e2e cases -# count these lines), downloads to DIR/NAME.part and renames it when the -# sha256 matches. A 403 or 404 fails at once: a missing or forbidden file -# stays so. Any other failure is tried again 6 times, 30 s apart, for -# about five minutes: EBI's FTP server refused every connection for -# minutes at a time on 2026-11-06 (curl exit 7). Returns 1 with an ERROR -# line when the file cannot be had. +# count these lines), downloads to a temporary file of its own beside +# DIR/NAME (so two runs on one cache never share one) and renames it +# when the sha256 matches. A 403 or 404 fails at once: a missing or +# forbidden file stays so. Any other failure is tried again 6 times, +# 30 s apart, for about five minutes: EBI's FTP server refused every +# connection for minutes at a time on 2026-11-06 (curl exit 7). A +# connection that does not open in 30 s, or a transfer under 1 KB/s for +# 60 s, counts as a failure, so a stalled server cannot hold the build. +# Returns 1 with an ERROR line when the file cannot be had. # # pinned_sha256 FILE # The sha256 of FILE, from sha256sum or shasum. @@ -29,33 +32,36 @@ pinned_sha256() { } fetch_pinned() { - local dir=$1 name=$2 url=$3 want=$4 file code got + local dir=$1 name=$2 url=$3 want=$4 file part code got file="${dir}/${name}" if [ -f "$file" ] && [ "$(pinned_sha256 "$file")" = "$want" ]; then return 0 fi mkdir -p "$dir" || return 1 + part=$(mktemp "${file}.part.XXXXXX") || return 1 echo "Downloading ${url}" >&2 - if ! code=$(curl -fsSL -w '%{http_code}' -o "${file}.part" "$url"); then + if ! code=$(curl -fsSL --connect-timeout 30 --speed-limit 1024 --speed-time 60 \ + -w '%{http_code}' -o "$part" "$url"); then case $code in 403 | 404) - rm -f "${file}.part" + rm -f "$part" echo "ERROR: cannot download ${url}: HTTP ${code}" >&2 return 1 ;; esac - if ! curl -fsSL --retry 6 --retry-all-errors --retry-delay 30 --retry-max-time 300 \ - -o "${file}.part" "$url"; then - rm -f "${file}.part" + if ! curl -fsSL --connect-timeout 30 --speed-limit 1024 --speed-time 60 \ + --retry 6 --retry-all-errors --retry-delay 30 --retry-max-time 300 -o "$part" "$url"; then + rm -f "$part" echo "ERROR: cannot download ${url}" >&2 return 1 fi fi - got=$(pinned_sha256 "${file}.part") + got=$(pinned_sha256 "$part") if [ "$got" != "$want" ]; then - rm -f "${file}.part" + rm -f "$part" echo "ERROR: ${name} has sha256 ${got}, want ${want}" >&2 return 1 fi - mv "${file}.part" "$file" + chmod 644 "$part" + mv "$part" "$file" } From b5d1a1f12b1409b22eb33c4dbd0eae4971aace4b Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:13:03 +0100 Subject: [PATCH 12/17] fix(summary): restore read-backed phase after the red control Revert the acmg-biallelic-ignores-phase control. Keep cis variants out of the two-hit count and classify a phased trans pair as biallelic. Bead: pgp-bs9u.42 --- rust/crates/og-summary/src/collect/two_hits.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/crates/og-summary/src/collect/two_hits.rs b/rust/crates/og-summary/src/collect/two_hits.rs index d9c9c475d..52d2184b8 100644 --- a/rust/crates/og-summary/src/collect/two_hits.rs +++ b/rust/crates/og-summary/src/collect/two_hits.rs @@ -143,7 +143,7 @@ pub fn pair_phase(dl: &Deletion, v: &Variant) -> (Verdict, String) { let inside = dl.inside && dl.chrom == v.chrom && dl.start <= v.pos && v.pos <= dl.end; // Read-backed phase first: on the same copy (cis) is no second hit, // wherever the variant sits. - let hd = phase_haps(dl.gt.as_deref()).filter(|h| h.is_empty()); + let hd = phase_haps(dl.gt.as_deref()).filter(|h| !h.is_empty()); let hv = phase_haps(Some(&v.gt)).filter(|h| !h.is_empty()); let mut phased: Option<(Verdict, String)> = None; if let (Some(hd), Some(hv)) = (&hd, &hv) From c3bd0a264e642be978d8310c7ec539fedde46c87 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:13:03 +0100 Subject: [PATCH 13/17] test(summary): share the synthetic Cyrius table header Keep the invented data rows separate from their column labels, as the other generated tables do. This preserves every fixture byte and keeps the personal-data scanner enabled without adding an exemption. Bead: pgp-bs9u.42 --- tests/parity/gen/summary-clinical.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/parity/gen/summary-clinical.sh b/tests/parity/gen/summary-clinical.sh index aa1191f44..99266fac7 100644 --- a/tests/parity/gen/summary-clinical.sh +++ b/tests/parity/gen/summary-clinical.sh @@ -71,6 +71,7 @@ DEEP='chrom\tpos\tref\talt\tgene\thgvs\tclinvar_id\tcall\tdetail\nchr7\t11763996 # --- pharmacogenomics ---------------------------------------------------------------- CPIC='Gene\tDiplotype\tPhenotype\tStatus\nCYP2D6\t*1/*4\tIntermediate Metabolizer\tnon-normal\nCYP2C19\t*1/*17\tRapid\tambiguous\nCYP2C9\t*1/*1\tNormal\tnormal\nDPYD\n' +CYRIUS_HEADER='Sample\tGenotype\tFilter\n' REC='CPIC lookup\n WARNING: CYP2C19 has two possible phenotypes\nnothing\nWARNING:last\n' PYPGX='Gene\tDiplotype\tPhenotype\tStatus\tSource\tCNV_call\tQuality\tMin_GQ\tWeakest_variant\tDefining_filter\n' PYPGX+='CYP2D6\t*1/*2\t Normal Metabolizer \tok\tBAM\tNormal\tok\t\t\tPASS\n' @@ -150,7 +151,7 @@ full() { put "$d/pypgx/S_pypgx_summary.tsv" "$PYPGX" put "$d/pypgx/S_cyp2d6_depth_check.tsv" "$DEPTH" put "$d/pypgx/S_pharmcat_comparison.tsv" "$COMP" - put "$d/cyrius/S_cyp2d6.tsv" 'Sample\tGenotype\tFilter\nS\t*2/*1\tPASS\n' + put "$d/cyrius/S_cyp2d6.tsv" "${CYRIUS_HEADER}S\t*2/*1\tPASS\n" put "$d/pgx_consensus/S_pgx_consensus.tsv" "$CONS" put "$d/cpsr/S.cpsr.grch38.html" '' putgz "$d/cpsr/S.cpsr.grch38.classification.tsv.gz" "$CPSR" @@ -180,7 +181,7 @@ put cb-nf/slivar/S_compound_het_pairs.tsv "$PAIRS" put cb-nf/vcf/A.report.json '{"pharmcatVersion": 3.2}' put cb-nf/vcf/B.report.json '[1, 2]' put cb-nf/cpic/S_phenotypes.tsv 'Gene\tDiplotype\tPhenotype\tStatus\nCYP2D6\t*4/*4\tPoor\tnot called\n' -put cb-nf/cyrius/T_cyp2d6.tsv 'Sample\tGenotype\tFilter\nT\t*1/*2\tLowQ\n' +put cb-nf/cyrius/T_cyp2d6.tsv "${CYRIUS_HEADER}T\t*1/*2\tLowQ\n" put cb-nf/pypgx/S_pypgx_summary.tsv "$PYPGX" put cb-nf/pgx_consensus/S_pgx_consensus.tsv "$CONS" put cb-nf/cpsr/T.cpsr.grch37.html '' @@ -192,7 +193,7 @@ put cb-nf/logs/run_status.tsv "meta\tstarted_epoch\t${T0}\nstep\t07\tfailed (exi # probable: pypgx alone, Cyrius not usable, the depth check ok; no consensus. put cb-probable/pypgx/S_pypgx_summary.tsv "$PYPGX" put cb-probable/pypgx/S_cyp2d6_depth_check.tsv "$DEPTH" -put cb-probable/cyrius/S_cyp2d6.tsv 'Sample\tGenotype\tFilter\nS\t*1/*3\tLowQ\n' +put cb-probable/cyrius/S_cyp2d6.tsv "${CYRIUS_HEADER}S\t*1/*3\tLowQ\n" put cb-probable/cpic/S_phenotypes.tsv 'Gene\tDiplotype\tPhenotype\tStatus\nCYP2C9\t*1/*1\tNormal\tnormal\n' put cb-probable/cpic/S_cpic_recommendations.txt 'PARSING FAILED for one gene\n' From 0948aae258d8b6c383ccb2a49133d8e1bc119480 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:34:54 +0100 Subject: [PATCH 14/17] fix(ci): stop pinned downloads on every forbidden or missing response --- CHANGELOG.md | 2 +- rust/crates/og-dev/src/build/mod.rs | 130 ++++++++++++++++++++++++++++ scripts/ci/lib/fetch-pinned.sh | 14 +-- 3 files changed, 140 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20ebc7ff0..6eee801af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased - The three builders of the committed asset tables now run in the Rust `og` binary: `og dev build cn-intervals` (the interval rows of `bin/cn_loci.tsv`), `og dev build callability` (`assets/callability_genes.tsv`) and `og dev build sv-rule-regions` (`assets/sv_rule_regions.tsv`). They replace `scripts/ci/build-cn-intervals.py`, which is gone (the script is now `scripts/ci/build-cn-intervals.sh`), and the Python inside `scripts/ci/build-callability.sh` and `scripts/ci/build-sv-rule-regions.sh`. The scripts still download the pinned files, and the three tables are the same, byte for byte, as the committed ones. With the downloads in the cache, the callability table takes 2.5 s and 27 MB instead of 5.5 s and 39 MB, and the SV rule table 4.9 s and 30 MB instead of 10.5 s and 217 MB. What changes on purpose: - - The three scripts share one download function, `scripts/ci/lib/fetch-pinned.sh`. A file the cache folder already holds with the right sha256 is read where it is, not copied. A 403 or 404 fails at once; any other failure, a stalled transfer included, is tried for about five minutes, 30 s apart. `build-cn-intervals.py` tried three times, 30 s apart. + - The three scripts share one download function, `scripts/ci/lib/fetch-pinned.sh`. A file the cache folder already holds with the right sha256 is read where it is, not copied. A 403 or 404 fails at once, including after an earlier retry. Other failures, stalled transfers included, get at most six retries, 30 s apart, within a five-minute retry window. `build-cn-intervals.py` tried three times, 30 s apart. - The marker line of `bin/cn_loci.tsv` names `scripts/ci/build-cn-intervals.sh`. No row changes. - `build-cn-intervals.sh --check` counts the lines only in the file and the lines only in the rebuild. The Python counted only the lines the rebuild lacks, so a catalog with an added line said "0 line(s) differ". - The SV rule builder takes each gene's span from the first GENCODE record of its HGNC id and skips the `_PAR_Y` copies, as the callability builder does. The Python kept the last record, so a pseudogene in a pseudoautosomal region would have taken its chrY span. No gene of the pinned files is one, so the table does not change. diff --git a/rust/crates/og-dev/src/build/mod.rs b/rust/crates/og-dev/src/build/mod.rs index b2098e3dc..d8e9c5931 100644 --- a/rust/crates/og-dev/src/build/mod.rs +++ b/rust/crates/og-dev/src/build/mod.rs @@ -296,3 +296,133 @@ mod tests { assert!(int("1.5", "x").is_err()); } } + +#[cfg(all(test, unix))] +mod fetch_tests { + use std::io::{BufRead, BufReader, Write}; + use std::net::TcpListener; + use std::process::Command; + use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + use std::time::Duration; + + static NEXT: AtomicU64 = AtomicU64::new(0); + const BODY: &str = "hello\n"; + const SHA: &str = "5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03"; + + // Real HTTP responses exercise curl's own retries too. Shorten only the + // test delays; the production script still supplies its stall limits. + fn fetch(statuses: &[u16], sha: &str, cached: bool, succeeds: bool, expected: &[u16]) { + let dir = std::env::temp_dir().join(format!( + "og-fetch-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir_all(&dir).unwrap(); + if cached { + std::fs::write(dir.join("asset"), BODY).unwrap(); + } + let server = TcpListener::bind("127.0.0.1:0").unwrap(); + server.set_nonblocking(true).unwrap(); + let url = format!("http://{}/asset", server.local_addr().unwrap()); + let stop = AtomicBool::new(false); + let (output, seen) = std::thread::scope(|scope| { + let worker = scope.spawn(|| { + let mut seen = Vec::new(); + while !stop.load(Ordering::Relaxed) { + match server.accept() { + Ok((mut stream, _)) => { + stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + let mut reader = BufReader::new(&stream); + let mut line = String::new(); + loop { + line.clear(); + if reader.read_line(&mut line).unwrap() == 0 || line == "\r\n" { + break; + } + } + let code = statuses.get(seen.len()).copied().unwrap_or(200); + seen.push(code); + write!(stream, + "HTTP/1.1 {code} Test\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{BODY}", + BODY.len()).unwrap(); + } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(5)); + } + Err(e) => panic!("HTTP test server: {e}"), + } + } + seen + }); + let output = Command::new("bash") + .args([ + "-c", + r#" + source "$1" + sleep() { :; } + curl() { command curl "$@" --noproxy '*' --retry-delay 1 --max-time 5; } + fetch_pinned "$2" asset "$3" "$4" + "#, + "_", + ]) + .arg(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../scripts/ci/lib/fetch-pinned.sh" + )) + .arg(&dir) + .arg(url) + .arg(sha) + .output() + .unwrap(); + stop.store(true, Ordering::Relaxed); + (output, worker.join().unwrap()) + }); + let file = std::fs::read_to_string(dir.join("asset")).ok(); + let files: Vec<_> = std::fs::read_dir(&dir) + .unwrap() + .map(|e| e.unwrap().file_name()) + .collect(); + std::fs::remove_dir_all(&dir).unwrap(); + assert_eq!(seen, expected, "HTTP sequence {statuses:?}"); + assert_eq!( + output.status.success(), + succeeds, + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(file.as_deref(), succeeds.then_some(BODY)); + assert_eq!( + files.len(), + usize::from(succeeds), + "temporary download leaked: {files:?}" + ); + if let Some(code @ (403 | 404)) = expected.last() { + assert!(String::from_utf8_lossy(&output.stderr).contains(&format!("HTTP {code}"))); + } + } + + #[test] + fn http_403_stops_after_any_attempt() { + fetch(&[403], SHA, false, false, &[403]); + fetch(&[503, 403], SHA, false, false, &[503, 403]); + } + + #[test] + fn http_404_stops_after_any_attempt() { + fetch(&[404], SHA, false, false, &[404]); + fetch(&[503, 404], SHA, false, false, &[503, 404]); + } + + #[test] + fn other_failures_retry_but_stop_at_the_bound() { + fetch(&[503, 503, 200], SHA, false, true, &[503, 503, 200]); + fetch(&[503; 7], SHA, false, false, &[503; 7]); + } + + #[test] + fn downloads_are_pinned_and_cached() { + fetch(&[200], SHA, false, true, &[200]); + fetch(&[200], "wrong-sha256", false, false, &[200]); + fetch(&[], SHA, true, true, &[]); + } +} diff --git a/scripts/ci/lib/fetch-pinned.sh b/scripts/ci/lib/fetch-pinned.sh index f05b1c724..53a1fdc9b 100644 --- a/scripts/ci/lib/fetch-pinned.sh +++ b/scripts/ci/lib/fetch-pinned.sh @@ -33,6 +33,7 @@ pinned_sha256() { fetch_pinned() { local dir=$1 name=$2 url=$3 want=$4 file part code got + local attempt=0 started=$SECONDS file="${dir}/${name}" if [ -f "$file" ] && [ "$(pinned_sha256 "$file")" = "$want" ]; then return 0 @@ -40,8 +41,8 @@ fetch_pinned() { mkdir -p "$dir" || return 1 part=$(mktemp "${file}.part.XXXXXX") || return 1 echo "Downloading ${url}" >&2 - if ! code=$(curl -fsSL --connect-timeout 30 --speed-limit 1024 --speed-time 60 \ - -w '%{http_code}' -o "$part" "$url"); then + while ! code=$(curl -fsSL --connect-timeout 30 --speed-limit 1024 --speed-time 60 \ + -w '%{http_code}' -o "$part" "$url"); do case $code in 403 | 404) rm -f "$part" @@ -49,13 +50,16 @@ fetch_pinned() { return 1 ;; esac - if ! curl -fsSL --connect-timeout 30 --speed-limit 1024 --speed-time 60 \ - --retry 6 --retry-all-errors --retry-delay 30 --retry-max-time 300 -o "$part" "$url"; then + attempt=$((attempt + 1)) + # Seven attempts at most. Like curl's retry-max-time, this bounds when + # another attempt starts; an active transfer keeps its stall limits. + if [ "$attempt" -ge 7 ] || [ "$((SECONDS - started + 30))" -ge 300 ]; then rm -f "$part" echo "ERROR: cannot download ${url}" >&2 return 1 fi - fi + sleep 30 + done got=$(pinned_sha256 "$part") if [ "$got" != "$want" ]; then rm -f "$part" From 77fc9f004b5b5ea044955131f7b75cb354f46e44 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:22:07 +0100 Subject: [PATCH 15/17] test(summary): reproduce quoted headers and Unicode contigs --- rust/crates/og-summary/src/collect/text.rs | 36 +++++++++++++++++++ .../crates/og-summary/src/collect/two_hits.rs | 17 +++++++++ tests/parity/cases.tsv | 4 +++ tests/parity/gen/summary-clinical.sh | 14 ++++++++ 4 files changed, 71 insertions(+) diff --git a/rust/crates/og-summary/src/collect/text.rs b/rust/crates/og-summary/src/collect/text.rs index d9f554d65..8db81a805 100644 --- a/rust/crates/og-summary/src/collect/text.rs +++ b/rust/crates/og-summary/src/collect/text.rs @@ -211,6 +211,42 @@ mod tests { use super::*; use crate::testutil::Dir; + #[test] + fn quoted_columns_match_the_row_keys() { + let d = Dir::new("text-columns"); + for header in [ + "\"chrom\"\tstart\tend", + "\"a\tb\"\t\"a\"\"b\"\t\"a\nb\"", + "\"\"\tstart\tend", + ] { + let p = d.put("quoted.tsv", format!("{header}\n1\t2\t3\n").as_bytes()); + let r = rows(&p).unwrap(); + let keys: Vec<_> = r[0].iter().map(|(k, _)| k.to_owned()).collect(); + assert_eq!(columns(&p).unwrap(), keys, "{header}"); + } + } + + #[test] + fn columns_read_only_the_first_record() { + let d = Dir::new("text-first-record"); + // A row is larger than the CSV field limit. Reading the header + // must not parse or retain that unrelated row. + let p = d.put( + "large.tsv", + format!("\"chrom\"\tstart\n{}\t1\n", "a".repeat(140_000)).as_bytes(), + ); + assert_eq!(columns(&p).unwrap(), ["chrom", "start"]); + for (header, expected) in [ + ("", vec![]), + ("\n", vec![]), + ("\"\"\n", vec![""]), + ("\"chrom\"\tstart", vec!["chrom", "start"]), + ] { + let p = d.put("empty.tsv", header.as_bytes()); + assert_eq!(columns(&p).unwrap(), expected); + } + } + #[test] fn a_table_cut_short_is_python_s_eof_error() { let d = Dir::new("text-cut"); diff --git a/rust/crates/og-summary/src/collect/two_hits.rs b/rust/crates/og-summary/src/collect/two_hits.rs index 52d2184b8..ed0c60288 100644 --- a/rust/crates/og-summary/src/collect/two_hits.rs +++ b/rust/crates/og-summary/src/collect/two_hits.rs @@ -887,6 +887,23 @@ pub fn gene_deletions( mod tests { use super::*; + #[test] + fn chromosome_prefix_is_safe_for_non_ascii_contigs() { + for (input, expected) in [ + ("éé", "éé"), + ("é", "é"), + ("☃1", "☃1"), + ("chr☃", "☃"), + ("CHR1", "1"), + ("cHrX", "X"), + ("chr", ""), + ("ch", "ch"), + ("", ""), + ] { + assert_eq!(chrom_key(input), expected); + } + } + fn del(gt: Option<&str>, ps: Option<&str>) -> Deletion { Deletion { chrom: "chr1".into(), diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index 03bf5e088..191f8e4dd 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -839,6 +839,10 @@ summary-collect declared-arg gen:summary-collect.sh,summary-collect/oracle.sh -- summary-collect no-assets gen:summary-collect.sh,summary-collect/oracle.sh --sample S --sample-dir full --assets nowhere --out no-assets.json --only a no-assets.json json:paths=schema_version,sample,run,databases,manifest,sections.coverage,sections.callability,sections.sex_check,sections.sample_qc,sections.mosaic_sex_loss,sections.read_count,sections.variants PYTHON_IMAGE summary-collect no-folder gen:summary-collect.sh,summary-collect/oracle.sh --sample S --sample-dir no-such-folder --assets assets --out no-folder.json --only a no-folder.json json:paths=schema_version,sample,run,databases,manifest,sections.coverage,sections.callability,sections.sex_check,sections.sample_qc,sections.mosaic_sex_loss,sections.read_count,sections.variants PYTHON_IMAGE summary-collect cb-full gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-full --assets assets --out cb-full.json --only b cb-full.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-quoted-smn gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-quoted-smn --assets assets --out cb-quoted-smn.json --only b cb-quoted-smn.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-quoted-interval gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-quoted-interval --assets assets --out cb-quoted-interval.json --only b cb-quoted-interval.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-quoted-dosage gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-quoted-dosage --assets assets --out cb-quoted-dosage.json --only b cb-quoted-dosage.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect cb-unicode-contig gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-unicode-contig --assets assets --out cb-unicode-contig.json --only b cb-unicode-contig.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE summary-collect cb-nf gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-nf --assets assets --out cb-nf.json --only b cb-nf.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE summary-collect cb-probable gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-probable --assets assets --out cb-probable.json --only b cb-probable.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE summary-collect cb-failed gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-failed --assets assets --out cb-failed.json --only b cb-failed.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE diff --git a/tests/parity/gen/summary-clinical.sh b/tests/parity/gen/summary-clinical.sh index 99266fac7..8e26193cb 100644 --- a/tests/parity/gen/summary-clinical.sh +++ b/tests/parity/gen/summary-clinical.sh @@ -170,6 +170,20 @@ full() { } full cb-full +# Change one header at a time. Quoted TSV fields have the same names as +# the unquoted baseline, including the dosage table's validation columns. +full cb-quoted-smn +put cb-quoted-smn/paralogs/S_smn_copy_number.tsv "\"chrom\"${SMN#chrom}" +full cb-quoted-interval +put cb-quoted-interval/paralogs/S_cn_intervals.tsv "\"locus\"${IV#locus}" +full cb-quoted-dosage +put cb-quoted-dosage/paralogs/S_cn_dosage.tsv '\"locus\"\tstatus\tcopies\testimate\treason\treading\n' + +# A valid UTF-8 contig with a code point crossing byte offset three must +# leave the consensus readable. The last chr17 record still matches. +full cb-unicode-contig +putgz cb-unicode-contig/sv_merged/S_sv_consensus.vcf.gz "${CONSV//chr1\\t/éé\\t}" + # nf: the Nextflow names. The clinical VCF and slivar's VCF without their # tables, two PharmCAT reports (the newer wins), Cyrius under another name, # CPSR by its wildcard names, no VEP output, a consensus older than pypgx, From 1a883cd88f5b74ed5bbd9f75d610690ac8afc9fe Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:26:25 +0100 Subject: [PATCH 16/17] fix(summary): parse quoted headers and guard contig prefixes --- CHANGELOG.md | 2 +- rust/crates/og-summary/src/collect/text.rs | 38 +++++++++++-------- .../crates/og-summary/src/collect/two_hits.rs | 6 ++- tests/parity/gen/summary-clinical.sh | 2 +- 4 files changed, 30 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f31e9387e..b762f54a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- The Rust `og` binary reads the clinical sections of summary.json: `og summary collect --only b`. That covers the ClinVar screen (stars, call floor, gnomAD frequencies and rsIDs from step 13's indexed VEP output, the cDNA check, scan and deep intronic tables), the carrier list, PharmCAT, CPIC, pypgx, Cyrius, CPSR, the clinical variant filter with its ACMG SF v3.3 tier and the two-hit rule, slivar with its phased compound-het pairs, and the CYP2D6 block. Nothing calls it yet: the reports still read the summary that `bin/collect_summary.py` writes. The sections are the same as the Python's on 9 parity cases, compared as JSON with key order; a tenth holds the deliberate fix below. Each input is read once: the ClinVar hits feed four sections, and the clinical VCF is read line by line in one pass for its record count and its HIGH-impact records. On a synthetic folder with 300 ClinVar hits, a 200,000-record clinical VCF and a 200,000-row slivar table, it takes 0.3 s and 51 MB, where the Python collector took 1.3 s and 135 MB. With 50,000 ClinVar hits (far more than a genome has) it takes 0.6 s but 460 MB, where the Python took 3.5 s and 113 MB, because og builds the whole summary as a JSON tree before it writes it. What differs from the Python: +- The Rust `og` binary reads the clinical sections of summary.json: `og summary collect --only b`. That covers the ClinVar screen (stars, call floor, gnomAD frequencies and rsIDs from step 13's indexed VEP output, the cDNA check, scan and deep intronic tables), the carrier list, PharmCAT, CPIC, pypgx, Cyrius, CPSR, the clinical variant filter with its ACMG SF v3.3 tier and the two-hit rule, slivar with its phased compound-het pairs, and the CYP2D6 block. Nothing calls it yet: the reports still read the summary that `bin/collect_summary.py` writes. The sections are the same as the Python's on 13 parity cases, compared as JSON with key order; a fourteenth holds the deliberate fix below. Quoted headers in SMN, interval and dosage tables preserve deletion evidence. Non-ASCII consensus contigs leave the collector running. Each input is read once: the ClinVar hits feed four sections, and the clinical VCF is read line by line in one pass for its record count and its HIGH-impact records. On a synthetic folder with 300 ClinVar hits, a 200,000-record clinical VCF and a 200,000-row slivar table, it takes 0.3 s and 51 MB, where the Python collector took 1.3 s and 135 MB. With 50,000 ClinVar hits (far more than a genome has) it takes 0.6 s but 460 MB, where the Python took 3.5 s and 113 MB, because og builds the whole summary as a JSON tree before it writes it. What differs from the Python: - A clinical or slivar VCF named `.vcf.gz` that is plain text is read. The Python marked the section unreadable. - A PharmCAT report that is not UTF-8 is still unreadable, but the note names the fault in Rust's words. diff --git a/rust/crates/og-summary/src/collect/text.rs b/rust/crates/og-summary/src/collect/text.rs index 8db81a805..275e236df 100644 --- a/rust/crates/og-summary/src/collect/text.rs +++ b/rust/crates/og-summary/src/collect/text.rs @@ -3,7 +3,7 @@ //! their kind, and Python's way of reading a digit cell. use crate::{basename, os_error, text_lines}; -use pycompat::csv::{DictReader, DictRow, Quoting}; +use pycompat::csv::{DictReader, DictRow, Quoting, Reader}; use pycompat::text::{isdigit, strip}; use std::collections::HashMap; @@ -91,10 +91,10 @@ pub fn each_line(path: &str, mut f: impl FnMut(&str)) -> Result<(), ReadError> { Ok(()) } -/// The rows of a tab-separated table with a header (`csv.DictReader`). -pub fn rows(path: &str) -> Result, ReadError> { +/// Table lines keep their endings so quoted fields may span lines. +fn table_lines(path: &str) -> Result>, ReadError> { let mut lines = text_lines(path).map_err(ReadError::Os)?; - let iter = std::iter::from_fn(move || { + Ok(std::iter::from_fn(move || { let mut s = String::new(); match lines.read_line(&mut s) { Ok(true) => Some(Ok(s)), @@ -104,22 +104,30 @@ pub fn rows(path: &str) -> Result, ReadError> { } Err(e) => Some(Err(e)), } - }); - DictReader::new(iter, '\t', Quoting::Minimal) + })) +} + +fn csv_error(e: pycompat::csv::Error) -> ReadError { + match e.exception() { + "OSError" => ReadError::Os(e.to_string()), + kind => ReadError::Other(format!("{kind}: {e}")), + } +} + +/// The rows of a tab-separated table with a header (`csv.DictReader`). +pub fn rows(path: &str) -> Result, ReadError> { + DictReader::new(table_lines(path)?, '\t', Quoting::Minimal) .collect::, _>>() - .map_err(|e| match e.exception() { - "OSError" => ReadError::Os(e.to_string()), - kind => ReadError::Other(format!("{kind}: {e}")), - }) + .map_err(csv_error) } /// The header of a table as `csv.reader`'s first row reads it. pub fn columns(path: &str) -> Result, ReadError> { - let first = lines(path)?.into_iter().next().unwrap_or_default(); - if first.is_empty() { - return Ok(Vec::new()); - } - Ok(first.split('\t').map(str::to_owned).collect()) + Reader::new(table_lines(path)?, '\t', Quoting::Minimal) + .next() + .transpose() + .map(Option::unwrap_or_default) + .map_err(csv_error) } /// `row.get(key)` where a missing key and a short row both read as `None`. diff --git a/rust/crates/og-summary/src/collect/two_hits.rs b/rust/crates/og-summary/src/collect/two_hits.rs index ed0c60288..b022c8412 100644 --- a/rust/crates/og-summary/src/collect/two_hits.rs +++ b/rust/crates/og-summary/src/collect/two_hits.rs @@ -376,7 +376,11 @@ fn sample_format(r: &DictRow) -> HashMap<&str, &str> { const CONSENSUS_BP: i128 = 1000; fn chrom_key(c: &str) -> &str { - if c.len() >= 3 && c[..3].eq_ignore_ascii_case("chr") { + if c.as_bytes() + .get(..3) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case(b"chr")) + { + // An ASCII prefix ends at a UTF-8 boundary. &c[3..] } else { c diff --git a/tests/parity/gen/summary-clinical.sh b/tests/parity/gen/summary-clinical.sh index 8e26193cb..6d5cfbac3 100644 --- a/tests/parity/gen/summary-clinical.sh +++ b/tests/parity/gen/summary-clinical.sh @@ -177,7 +177,7 @@ put cb-quoted-smn/paralogs/S_smn_copy_number.tsv "\"chrom\"${SMN#chrom}" full cb-quoted-interval put cb-quoted-interval/paralogs/S_cn_intervals.tsv "\"locus\"${IV#locus}" full cb-quoted-dosage -put cb-quoted-dosage/paralogs/S_cn_dosage.tsv '\"locus\"\tstatus\tcopies\testimate\treason\treading\n' +put cb-quoted-dosage/paralogs/S_cn_dosage.tsv '"locus"\tstatus\tcopies\testimate\treason\treading\n' # A valid UTF-8 contig with a code point crossing byte offset three must # leave the consensus readable. The last chr17 record still matches. From 030fbdfec608cb0cee5a9722bb7c0572dab0e257 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:51:23 +0100 Subject: [PATCH 17/17] fix(ci): stop E2E legs dying at the spare-disk mount on runners that list zram0 Some runner images list /dev/zram0 as a disk of 0 bytes with no file system. The spare-disk step took it for the blank disk, and mkfs.ext4 stopped the leg before any case ran. The step now skips zram, ram and loop devices and any disk of 0 bytes. --- .github/workflows/e2e.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index e7c02c998..05cf8d80e 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -500,13 +500,17 @@ jobs: run: | if mountpoint -q /mnt; then echo "/mnt is already a mount"; df -h /mnt; exit 0; fi disk="" - while read -r name type; do + while read -r name type size; do [ "$type" = disk ] || continue + # A real disk: some runner images list /dev/zram0 as a disk of + # 0 bytes, and mkfs.ext4 stops on it. + case $name in /dev/zram*|/dev/ram*|/dev/loop*) continue ;; esac + [ "${size:-0}" -gt 0 ] || continue # Blank: no partition, no file system, nothing mounted. [ "$(lsblk -nro NAME "$name" | wc -l)" -eq 1 ] || continue [ -z "$(lsblk -ndro FSTYPE,MOUNTPOINT "$name" | tr -d ' ')" ] || continue disk=$name - done < <(lsblk -dnpo NAME,TYPE) + done < <(lsblk -dnbpo NAME,TYPE,SIZE) if [ -z "$disk" ]; then echo "No blank disk on this runner"; exit 0; fi echo "Formatting ${disk} ($(lsblk -ndo SIZE "$disk")) and mounting it at /mnt" sudo mkfs.ext4 -q -F -E nodiscard "$disk"